From b495a86fb5c7dacdd740182f73e8b26dc1401401 Mon Sep 17 00:00:00 2001 From: Ruben Oanta Date: Tue, 10 Jul 2018 22:14:02 +0000 Subject: [PATCH 001/409] Twitter-oss: update OSS libraries post-release to 18.8.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 18.8.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D190776 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8d429f49..f7c5b480 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 4308c9ee..979cecf6 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "18.7.0" +val releaseVersion = "18.8.0-SNAPSHOT" val jacksonVersion = "2.8.4" val jacksonLibs = Seq( From 2b2140c617f90e8e100a8d45ac00637de86c0135 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Thu, 19 Jul 2018 03:57:43 +0000 Subject: [PATCH 002/409] finagle-core: Call Trace.isActivelyTracing less # Problem We call `isActivelyTracing` too often to prevent unnecessarily structures from materializing (a very simple proxy reports 12 calls/request). The overhead of `isActivelyTracing` (about 5% of CPU profile) may outweigh the benefits of saving allocations. Additionally, a canonical usage pattern requires users to guard tracing operations behind the `isActivelyTracing` result, which pretty much doubles the number of checks we do today. # Solution Do not check `isActivelyTracing` internally before materializing annotations and rely on callers checking that externally. This cuts the number of `isActivelyTracing` calls in a proxy in half: from 12 to 7. JIRA Issues: CSL-5625 Differential Revision: https://phabricator.twitter.biz/D193300 --- .../server/handler/TracingHandlerTest.scala | 32 ++++--------------- 1 file changed, 7 insertions(+), 25 deletions(-) diff --git a/server/src/test/scala/com/twitter/server/handler/TracingHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/TracingHandlerTest.scala index 706f9d23..6011517a 100644 --- a/server/src/test/scala/com/twitter/server/handler/TracingHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/TracingHandlerTest.scala @@ -1,11 +1,9 @@ package com.twitter.server.handler import com.twitter.finagle.http.{Request, Status} -import com.twitter.finagle.tracing.{Record, Trace, TraceId, Tracer} +import com.twitter.finagle.tracing.{Trace, Tracer} import com.twitter.util.Await import org.junit.runner.RunWith -import org.mockito.Matchers.any -import org.mockito.Mockito.{never, verify, when} import org.scalatest.junit.JUnitRunner import org.scalatest.mockito.MockitoSugar import org.scalatest.{BeforeAndAfter, FunSuite} @@ -16,20 +14,13 @@ class TracingHandlerTest extends FunSuite with MockitoSugar with BeforeAndAfter test("enable tracing") { val tracer = mock[Tracer] - when(tracer.isActivelyTracing(any[TraceId])).thenReturn(true) - try { - // FIXME: This relies on a global. Trace.disable() Trace.letTracer(tracer) { - Trace.record("msg") - verify(tracer, never()).record(any(classOf[Record])) - + assert(!Trace.enabled) val request = Request("/", ("enable", "true")) assert(Await.result(service(request)).status == Status.Ok) - - Trace.record("msg") - verify(tracer).record(any(classOf[Record])) + assert(Trace.enabled) } } finally { Trace.enable() @@ -38,22 +29,13 @@ class TracingHandlerTest extends FunSuite with MockitoSugar with BeforeAndAfter test("disable tracing") { val tracer = mock[Tracer] - when(tracer.isActivelyTracing(any[TraceId])).thenReturn(true) - try { Trace.enable() Trace.letTracer(tracer) { - Trace.record("msg") - verify(tracer).record(any(classOf[Record])) - - val tracer2 = mock[Tracer] - Trace.letTracer(tracer2) { - val request = Request("/", ("disable", "true")) - assert(Await.result(service(request)).status == Status.Ok) - - Trace.record("msg") - verify(tracer2, never()).record(any(classOf[Record])) - } + assert(Trace.enabled) + val request = Request("/", ("disable", "true")) + assert(Await.result(service(request)).status == Status.Ok) + assert(!Trace.enabled) } } finally { Trace.enable() From eb4532de0db9f42f235b410bbd2fab70dc9c2243 Mon Sep 17 00:00:00 2001 From: Neuman Vong Date: Thu, 26 Jul 2018 15:17:06 +0000 Subject: [PATCH 003/409] finagle-core: privatize type of Stack constructors Problem Leaf and Node allow public access to the internals of Stack via pattern matching. Solution Make Leaf and Node private. Turn the type constructors into methods that return `Stack[T]` instead of the underlying type. JIRA Issues: CSL-6758 Differential Revision: https://phabricator.twitter.biz/D195924 --- .../scala/com/twitter/server/view/StackRegistryViewTest.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/scala/com/twitter/server/view/StackRegistryViewTest.scala b/server/src/test/scala/com/twitter/server/view/StackRegistryViewTest.scala index cd34ddc7..570fbfe1 100644 --- a/server/src/test/scala/com/twitter/server/view/StackRegistryViewTest.scala +++ b/server/src/test/scala/com/twitter/server/view/StackRegistryViewTest.scala @@ -54,7 +54,7 @@ private[server] object StackRegistryViewTest { } } - val sb = new StackBuilder(Stack.Leaf[Int => Int](Stack.Role("identity"), identity[Int] _)) + val sb = new StackBuilder(Stack.leaf[Int => Int](Stack.Role("identity"), identity[Int] _)) sb.push(Baz.module) sb.push(Bar.module) sb.push(Foo.module) From 06836d399d8f6596fe001b64597d421633bfb0a4 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Fri, 3 Aug 2018 20:00:54 +0000 Subject: [PATCH 004/409] twitter-server: Add c.t.server.Hook#onExit lifecycle callback Problem There may be cases where a Hook needs to run in the `App#onExit` lifecycle. Solution Add an `onExit` callback function which can be implemented by a `c.t.server.Hook` and implement calling it in the Hooks trait which loads and fires service-loaded hooks. Result A `c.t.server.Hook` can define logic which will be triggered during the `onExit` lifecycle phase of an App. Differential Revision: https://phabricator.twitter.biz/D198379 --- CHANGES | 8 +++ .../main/scala/com/twitter/server/Hook.scala | 18 +++++ .../main/scala/com/twitter/server/Hooks.scala | 20 ++---- .../scala/com/twitter/server/NewHook.scala | 28 ++++++++ server/src/test/java/BUILD | 36 +++++----- server/src/test/scala/BUILD | 4 +- .../scala/com/twitter/server/HookTest.scala | 70 +++++++++++++++++++ .../twitter/server/TwitterServerTest.scala | 8 +-- 8 files changed, 154 insertions(+), 38 deletions(-) create mode 100644 server/src/main/scala/com/twitter/server/Hook.scala create mode 100644 server/src/main/scala/com/twitter/server/NewHook.scala create mode 100644 server/src/test/scala/com/twitter/server/HookTest.scala diff --git a/CHANGES b/CHANGES index 7bcb3dda..3b9be5f5 100644 --- a/CHANGES +++ b/CHANGES @@ -7,6 +7,14 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +New Features +~~~~~~~~~~~~ + + * Add `onExit` lifecycle callback to `c.t.server.Hook` (which is now an abstract class) to allow + implemented hooks to execute functions in the `App#onExit` lifecycle phase. Note: + `c.t.server.Hook#premain` now has a default implementation and requires the `override` modifier. + ``PHAB_ID=D198379`` + 18.7.0 ------- diff --git a/server/src/main/scala/com/twitter/server/Hook.scala b/server/src/main/scala/com/twitter/server/Hook.scala new file mode 100644 index 00000000..cf32a667 --- /dev/null +++ b/server/src/main/scala/com/twitter/server/Hook.scala @@ -0,0 +1,18 @@ +package com.twitter.server + +/** + * Defines a hook into an [[com.twitter.app.App]]. + */ +abstract class Hook { + + /** + * Executed in `App#premain` which is right before the user's main is invoked. + */ + def premain(): Unit = () + + /** + * Executed in `App#onExit` which is when shutdown is requested. All registered exit hooks + * run in parallel and are executed after all `App#postmain` functions complete. + */ + def onExit(): Unit = () +} \ No newline at end of file diff --git a/server/src/main/scala/com/twitter/server/Hooks.scala b/server/src/main/scala/com/twitter/server/Hooks.scala index 15e6a323..facd709a 100644 --- a/server/src/main/scala/com/twitter/server/Hooks.scala +++ b/server/src/main/scala/com/twitter/server/Hooks.scala @@ -3,24 +3,11 @@ package com.twitter.server import com.twitter.app.App import com.twitter.finagle.util.LoadService -/** - * Defines a hook into an App. - */ -trait Hook { - def premain(): Unit -} - -/** - * Create a new hook for the given App. NewHooks are - * service-loaded. - */ -trait NewHook extends (App => Hook) - /** * Mix-in to include service-loaded hooks. */ trait Hooks { self: App => - private val hooks = LoadService[NewHook]() map { newHook => + private val hooks = LoadService[NewHook]().map { newHook => newHook(self) } @@ -28,4 +15,7 @@ trait Hooks { self: App => for (hook <- hooks) hook.premain() } -} + + for (hook <- hooks) + onExit { hook.onExit() } +} \ No newline at end of file diff --git a/server/src/main/scala/com/twitter/server/NewHook.scala b/server/src/main/scala/com/twitter/server/NewHook.scala new file mode 100644 index 00000000..887e6933 --- /dev/null +++ b/server/src/main/scala/com/twitter/server/NewHook.scala @@ -0,0 +1,28 @@ +package com.twitter.server + +import com.twitter.app.App + +/** + * Create a new hook for the given App. NewHooks are service-loaded. + * + * To use, extend the [[NewHook]] trait and implement an apply method which + * returns a [[Hook]] implementation, e.g., + * + * Add the Hook as a service-loaded class in + * /META-INF/services/com.twitter.server.NewHook + * + * {{{ + * class MyHook extends NewHook { + * def apply(app: App) = new Hook { + * + * override def premain(): Unit = ??? + * + * override def onExit(): Unit = ??? + * } + * } + * }}} + * + * @see [[com.twitter.server.Hook]] + * @see [[com.twitter.finagle.util.LoadService]] + */ +trait NewHook extends (App => Hook) \ No newline at end of file diff --git a/server/src/test/java/BUILD b/server/src/test/java/BUILD index bfa24276..90f366d2 100644 --- a/server/src/test/java/BUILD +++ b/server/src/test/java/BUILD @@ -1,18 +1,20 @@ junit_tests( - dependencies=[ - '3rdparty/jvm/junit:junit', - '3rdparty/jvm/org/mockito:mockito-all', - '3rdparty/jvm/org/scala-lang:scala-library', - '3rdparty/jvm/org/scalacheck:scalacheck', - 'finagle/finagle-core/src/main/scala:scala', - 'finagle/finagle-http/src/main/scala', - 'twitter-server/server/src/main/scala', - 'util/util-app/src/main/scala:scala', - 'util/util-core/src/main/scala:scala', - 'util/util-lint/src/main/scala:scala', - 'util/util-slf4j-api/src/main/scala:scala', - 'util/util-stats/src/main/scala:scala', - ], - fatal_warnings=False, - sources=rglobs('*.java') -) + sources = rglobs("*.java"), + fatal_warnings = False, + dependencies = [ + "3rdparty/jvm/junit", + "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/scala-lang:scala-library", + "3rdparty/jvm/org/scalacheck", + "3rdparty/jvm/org/slf4j:slf4j-api", + "3rdparty/jvm/org/slf4j:slf4j-nop", + "finagle/finagle-core/src/main/scala", + "finagle/finagle-http/src/main/scala", + "twitter-server/server/src/main/scala", + "util/util-app/src/main/scala", + "util/util-core/src/main/scala", + "util/util-lint/src/main/scala", + "util/util-slf4j-api/src/main/scala", + "util/util-stats/src/main/scala", + ], +) \ No newline at end of file diff --git a/server/src/test/scala/BUILD b/server/src/test/scala/BUILD index 58f125fd..a58bf965 100644 --- a/server/src/test/scala/BUILD +++ b/server/src/test/scala/BUILD @@ -8,6 +8,8 @@ junit_tests( "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scalacheck", "3rdparty/jvm/org/scalatest", + "3rdparty/jvm/org/slf4j:slf4j-api", + "3rdparty/jvm/org/slf4j:slf4j-nop", "finagle/finagle-core/src/main/scala", "finagle/finagle-http/src/main/scala", "finagle/finagle-toggle", @@ -21,4 +23,4 @@ junit_tests( "util/util-stats/src/main/scala", "util/util-tunable/src/main/scala", ], -) +) \ No newline at end of file diff --git a/server/src/test/scala/com/twitter/server/HookTest.scala b/server/src/test/scala/com/twitter/server/HookTest.scala new file mode 100644 index 00000000..86c6c1af --- /dev/null +++ b/server/src/test/scala/com/twitter/server/HookTest.scala @@ -0,0 +1,70 @@ +package com.twitter.server + +import org.scalatest.FunSuite +import scala.collection.mutable + +class OnExitHook1 extends Hook { + override def onExit(): Unit = { + throw new Exception("onExit 1") + } +} + +class OnExitHook2 extends Hook { + override def onExit(): Unit = { + throw new Exception("onExit 2") + } +} + +class HookTestTwitterServer(hooks: Seq[Hook]) extends TwitterServer { + override val defaultAdminPort = 0 + + val bootstrapSeq: mutable.MutableList[Symbol] = mutable.MutableList.empty[Symbol] + + def main(): Unit = { + bootstrapSeq += 'Main + } + + override def exitOnError(throwable: Throwable): Unit = { + throw throwable + } + + init { + bootstrapSeq += 'Init + } + + premain { + bootstrapSeq += 'PreMain + for (hook <- hooks) + hook.premain() + } + + onExit { + bootstrapSeq += 'Exit + } + + for (hook <- hooks) + onExit { hook.onExit() } + + postmain { + bootstrapSeq += 'PostMain + } +} + +class HookTest extends FunSuite { + + val hooks: Seq[Hook] = Seq(new OnExitHook1(), new OnExitHook2()) + + test("Hooks which error in OnExit are properly captured in onExit blocks") { + val twitterServer = new HookTestTwitterServer(hooks) + + val e = intercept[Exception] { + twitterServer.main(args = Array.empty[String]) + } + + assert(e.getSuppressed.length == 2) + assert( + twitterServer.bootstrapSeq == + Seq('Init, 'PreMain, 'Main, 'PostMain, 'Exit) + ) + } +} \ No newline at end of file diff --git a/server/src/test/scala/com/twitter/server/TwitterServerTest.scala b/server/src/test/scala/com/twitter/server/TwitterServerTest.scala index 80053d68..50dfc82b 100644 --- a/server/src/test/scala/com/twitter/server/TwitterServerTest.scala +++ b/server/src/test/scala/com/twitter/server/TwitterServerTest.scala @@ -3,17 +3,15 @@ package com.twitter.server import com.twitter.finagle.Service import com.twitter.finagle.http.{Request, Response} import com.twitter.util._ -import java.net.{InetAddress, InetSocketAddress} import org.junit.runner.RunWith import org.scalatest.FunSuite import org.scalatest.junit.JUnitRunner import scala.collection.mutable class TestTwitterServer extends TwitterServer { - override val adminPort = - flag("admin.port", new InetSocketAddress(InetAddress.getLoopbackAddress, 0), "") + override val defaultAdminPort = 0 - val bootstrapSeq = mutable.MutableList.empty[Symbol] + val bootstrapSeq: mutable.MutableList[Symbol] = mutable.MutableList.empty[Symbol] def main(): Unit = { bootstrapSeq += 'Main @@ -70,4 +68,4 @@ class TwitterServerTest extends FunSuite { twitterServer.main(args = Array.empty[String]) assert(twitterServer.bootstrapSeq == Seq('Init, 'PreMain, 'Main, 'Exit, 'PostMain)) } -} +} \ No newline at end of file From 5038a9b9cfe426490fa6cbcb0bad513b9922acdc Mon Sep 17 00:00:00 2001 From: Daniel Schobel Date: Mon, 6 Aug 2018 21:22:26 +0000 Subject: [PATCH 005/409] Twitter-oss: Prepare OSS libraries for release 18.8 Problem We want to release the next versions of our Twitter OSS libraries 18.8 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-6773 Differential Revision: https://phabricator.twitter.biz/D199004 --- CHANGES | 3 +++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index 3b9be5f5..61d55e9e 100644 --- a/CHANGES +++ b/CHANGES @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +18.8.0 +------- + New Features ~~~~~~~~~~~~ diff --git a/README.md b/README.md index f7c5b480..8d429f49 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 979cecf6..119e7aa4 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "18.8.0-SNAPSHOT" +val releaseVersion = "18.8.0" val jacksonVersion = "2.8.4" val jacksonLibs = Seq( From d5cf1757583494559c5626b75ccdc2e8676393ab Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Tue, 7 Aug 2018 01:04:29 +0000 Subject: [PATCH 006/409] finagle|finatra|util: Introduce polymorphic Reader/Writer # Problem Both `Reader` and `Writer` are efficient and easy to reason about streaming APIs. Yet their usages are very limited given the fixed in/out types they operate on (`Buf`). # Solution Abstract both `Reader` and `Writer` over their produced/consumed type. ``` // before trait Reader { def read(): Future[Option[Buf]] } trait Writer { def write(b: Buf): Future[Unit] } // after trait Reader[+A] { def read(): Future[Option[A]] } trait Writer[-A] { def write(a: A): Future[Unit] } ``` JIRA Issues: CSL-6684 TBR=true Differential Revision: https://phabricator.twitter.biz/D195638 --- server/src/main/scala/com/twitter/server/view/IndexView.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/scala/com/twitter/server/view/IndexView.scala b/server/src/main/scala/com/twitter/server/view/IndexView.scala index fcd498d3..3ab8ee1f 100644 --- a/server/src/main/scala/com/twitter/server/view/IndexView.scala +++ b/server/src/main/scala/com/twitter/server/view/IndexView.scala @@ -27,7 +27,7 @@ object IndexView { } /** Render nav and contents into an html template. */ - def render(title: String, uri: String, nav: Seq[Entry], contents: Reader): Reader = { + def render(title: String, uri: String, nav: Seq[Entry], contents: Reader[Buf]): Reader[Buf] = { def renderNav( ls: Seq[Entry], From dc4f266bc1b0cd919cc8f4bfcae46a0cc6053cc0 Mon Sep 17 00:00:00 2001 From: Daniel Schobel Date: Tue, 7 Aug 2018 03:49:35 +0000 Subject: [PATCH 007/409] Twitter-oss: update OSS libraries post-release to 18.9.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 18.9.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D199228 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8d429f49..f7c5b480 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 119e7aa4..0cbfb2d9 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "18.8.0" +val releaseVersion = "18.9.0-SNAPSHOT" val jacksonVersion = "2.8.4" val jacksonLibs = Seq( From 4c193594053cd332ef2ff1602df82ad948024a7e Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Tue, 14 Aug 2018 18:40:19 +0000 Subject: [PATCH 008/409] twitter-server: Make parsed build properties available through object Problem It would be great to allow for other code in TwitterServer to have access to the parsed `build.properties` without needing to re-parse the file as the `c.t.server.handler.ServerInfoHandler` already does this. Solution Move the logic to parse the `build.properties` into a separate object with utility methods to expose access to the read properties. Update the `ServerInfoHandler` to use this object utility. Result Other TwitterServer code can get access to the read `build.properties` file without needing to re-parse the file. JIRA Issues: CSL-6665 Differential Revision: https://phabricator.twitter.biz/D201207 --- CHANGES | 9 +++ .../com/twitter/server/BuildProperties.scala | 74 +++++++++++++++++++ .../server/handler/ServerInfoHandler.scala | 46 +++--------- .../handler/ServerInfoHandlerTest.scala | 16 ++-- 4 files changed, 100 insertions(+), 45 deletions(-) create mode 100644 server/src/main/scala/com/twitter/server/BuildProperties.scala diff --git a/CHANGES b/CHANGES index 61d55e9e..3e75a5cb 100644 --- a/CHANGES +++ b/CHANGES @@ -7,6 +7,15 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Runtime Behavior Changes +~~~~~~~~~~~~~~~~~~~~~~~~ + + * Move logic to parse the server `build.properties` file out the `c.t.server.handler.ServerInfoHandler` + and into a utility object, `c.t.server.BuildProperties` to allow for accessing by other server + logic such that the properties do not need to be re-parsed anytime access is desired. Failure to + load the properties can result in the server not starting in the case of a Fatal exception + being thrown. ``PHAB_ID=D201207`` + 18.8.0 ------- diff --git a/server/src/main/scala/com/twitter/server/BuildProperties.scala b/server/src/main/scala/com/twitter/server/BuildProperties.scala new file mode 100644 index 00000000..b2fcda4f --- /dev/null +++ b/server/src/main/scala/com/twitter/server/BuildProperties.scala @@ -0,0 +1,74 @@ +package com.twitter.server + +import com.twitter.util.logging.Logger +import java.util.Properties +import scala.collection.JavaConverters._ +import scala.util.control.NonFatal + +/** + * A simply utility for loading information from a build.properties file. The ClassLoader for the + * given object is used to load the build.properties file, which is first searched for relative to + * the given object's class's package (class-package-name/build.properties), and if not found there, + * then it is searched for with an absolute path ("/build.properties"). + */ +private[server] object BuildProperties { + private[this] val log = Logger(BuildProperties.getClass) + + private[this] val basicServerInfo: Map[String, String] = + Map( + "name" -> "unknown", + "version" -> "0.0", + "build" -> "unknown", + "build_revision" -> "unknown", + "build_branch_name" -> "unknown", + "merge_base" -> "unknown", + "merge_base_commit_date" -> "unknown", + "scm_repository" -> "unknown" + ) + + private[this] val buildProperties = new Properties + try { + buildProperties.load(BuildProperties.getClass.getResource("build.properties").openStream) + } catch { + case NonFatal(_) => + try { + buildProperties.load(BuildProperties.getClass.getResource("/build.properties").openStream) + } catch { + case NonFatal(e) => + log.warn("Unable to load build.properties file from classpath. " + e.getMessage) + } + } + + /* Lazy to allow for loading of the buildProperties before this val is read */ + private[this] lazy val combinedInfo: Map[String, String] = + basicServerInfo ++ buildProperties.asScala + + /** + * Returns the [[String]] value associated with this key or a `NoSuchElementException` if there + * is no mapping from the given key to a value. + * + * @param key the key + * @return the value associated with the given key, or a `NoSuchElementException`. + */ + def get(key: String): String = { + all(key) + } + + /** + * Returns the value associated with a key, or a default value if the key is not contained in the map. + * + * @param key the key + * @param defaultValue a default value in case no binding for `key` is found in the map. + * @return the value associated with `key` if it exists, otherwise the `defaultValue`. + */ + def get(key: String, defaultValue: String): String = { + all.getOrElse(key, defaultValue) + } + + /** + * Return all build properties. + */ + def all: Map[String, String] = { + combinedInfo + } +} diff --git a/server/src/main/scala/com/twitter/server/handler/ServerInfoHandler.scala b/server/src/main/scala/com/twitter/server/handler/ServerInfoHandler.scala index fd70a062..3c37cdbe 100644 --- a/server/src/main/scala/com/twitter/server/handler/ServerInfoHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/ServerInfoHandler.scala @@ -1,56 +1,28 @@ package com.twitter.server.handler import com.twitter.finagle.Service -import com.twitter.finagle.stats.LoadedStatsReceiver import com.twitter.finagle.http.{Request, Response} +import com.twitter.finagle.stats.LoadedStatsReceiver import com.twitter.io.Buf +import com.twitter.server.BuildProperties import com.twitter.server.util.HttpUtils.newResponse import com.twitter.server.util.JsonConverter import com.twitter.util.Future import com.twitter.util.registry.GlobalRegistry import java.lang.management.ManagementFactory -import java.util.{Date, Properties} +import java.util.Date import scala.collection.JavaConverters._ /** * A simple http service for serving up information pulled from a build.properties - * file. The ClassLoader for the given object is used to load the build.properties file, - * which is first searched for relative to the given object's class's package - * (class-package-name/build.properties), and if not found there, then it is searched for - * with an absolute path ("/build.properties"). + * file. */ class ServerInfoHandler(obj: AnyRef) extends Service[Request, Response] { private[this] val mxRuntime = ManagementFactory.getRuntimeMXBean - private[this] val buildProperties = new Properties - - try { - buildProperties.load(obj.getClass.getResource("build.properties").openStream) - } catch { - case _: Throwable => - try { - buildProperties.load(obj.getClass.getResource("/build.properties").openStream) - } catch { - case _: Throwable => - } - } - - private[this] val basicServerInfo = Map( - "name" -> "unknown", - "version" -> "0.0", - "build" -> "unknown", - "build_revision" -> "unknown", - "build_branch_name" -> "unknown", - "merge_base" -> "unknown", - "merge_base_commit_date" -> "unknown", - "scm_repository" -> "unknown" - ) - - private[this] val combinedInfo = basicServerInfo ++ buildProperties.asScala - private[this] val registry = GlobalRegistry.get - combinedInfo.foreach { + BuildProperties.all.foreach { case (key, value) => registry.put(Seq("build.properties", key), value) } @@ -79,16 +51,16 @@ class ServerInfoHandler(obj: AnyRef) extends Service[Request, Response] { ) // Expose this build revision as a number. Useful to check cluster consistency. - combinedInfo.get("build.git.revision.number") match { + BuildProperties.all.get("build.git.revision.number") match { case Some(rev) => LoadedStatsReceiver.provideGauge("build.git.revision.number") { rev.toFloat } case None => } - private[this] val serverInfo = combinedInfo ++ Map( + private[this] val serverInfo = BuildProperties.all ++ Map( "build_last_few_commits" -> - buildProperties.getProperty("build_last_few_commits", "unknown").split("\n"), + BuildProperties.get("build_last_few_commits", "unknown").split("\n"), "build" -> - buildProperties.getProperty("build_name", "unknown"), + BuildProperties.get("build_name", "unknown"), "start_time" -> new Date(mxRuntime.getStartTime).toString ) - "build_name" diff --git a/server/src/test/scala/com/twitter/server/handler/ServerInfoHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/ServerInfoHandlerTest.scala index 11512bc7..bb348c8c 100644 --- a/server/src/test/scala/com/twitter/server/handler/ServerInfoHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/ServerInfoHandlerTest.scala @@ -20,21 +20,21 @@ class ServerInfoHandlerTest extends FunSuite { assert(res.status == Status.Ok) val info = res.contentString - assert(info contains ("\"build\" : \"unknown\"")) - assert(info contains ("\"build_revision\" : \"unknown\"")) - assert(info contains ("\"name\" : \"twitter-server\"")) - assert(info contains ("\"version\" : \"0.0.0\"")) - assert(info contains ("\"start_time\" :")) - assert(info contains ("\"uptime\" :")) + assert(info.contains("\"build\" : \"unknown\"")) + assert(info.contains("\"build_revision\" : \"unknown\"")) + assert(info.contains("\"name\" : \"twitter-server\"")) + assert(info.contains("\"version\" : \"0.0.0\"")) + assert(info.contains("\"start_time\" :")) + assert(info.contains("\"uptime\" :")) // user-defined properties - assert(info contains ("\"foo\" : \"bar\"")) + assert(info.contains("\"foo\" : \"bar\"")) } test("ServerInfo handler returns the right content-type") { val handler = new ServerInfoHandler(this) val req = Request("/") val res = Await.result(handler(req)) - assert(res.contentType == Some("application/json;charset=UTF-8")) + assert(res.contentType.contains("application/json;charset=UTF-8")) } for (key <- Seq(Seq("build.properties"), Seq("system", "properties"), Seq("system", "env"))) From f8c8e54c03d6932cbf611cc290dbe2a1f916554f Mon Sep 17 00:00:00 2001 From: Kevin Oliver Date: Fri, 24 Aug 2018 15:44:39 +0000 Subject: [PATCH 009/409] Rename CHANGES to CHANGELOG.rst Problem Github renders reStructuredText (rst) files. While our changelog is written in rst, because it does not have that as its file type it is rendered as plain text. Solution Rename to CHANGELOG.rst. This uses changelog instead of changes, which seems to be the more canonical name. Result Changelogs are rendered nicely in the Github UI. Differential Revision: https://phabricator.twitter.biz/D206432 --- CHANGES => CHANGELOG.rst | 0 CONTRIBUTING.md | 6 +++--- README.md | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) rename CHANGES => CHANGELOG.rst (100%) diff --git a/CHANGES b/CHANGELOG.rst similarity index 100% rename from CHANGES rename to CHANGELOG.rst diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1b4cc899..1c240650 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -86,8 +86,8 @@ compatibility can be kept by adding a forwarding method. Note that we avoid adding default arguments because this is not a compatible change for our Java users. However, when the benefits outweigh the costs, we are willing to break APIs. The break should be noted in the Breaking -API Changes section of the [changelog](CHANGES). Note that changes to -non-public APIs will not be called out in the [changelog](CHANGES). +API Changes section of the [changelog][changes]. Note that changes to +non-public APIs will not be called out in the [changelog][changes]. ## Java @@ -231,6 +231,6 @@ requests that improve the existing Scaladocs! [scalatest]: http://www.scalatest.org/ [ssg]: http://docs.scala-lang.org/style/scaladoc.html [travis-ci]: https://travis-ci.org/twitter/twitter-server -[changes]: https://github.com/twitter/twitter-server/blob/develop/CHANGES +[changes]: https://github.com/twitter/twitter-server/blob/develop/CHANGELOG.rst [scalacheck]: https://www.scalacheck.org/ [gendrivenprop]: http://www.scalatest.org/user_guide/generator_driven_property_checks diff --git a/README.md b/README.md index f7c5b480..9df46cb3 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Browse the [user guide](https://twitter.github.io/twitter-server/). [Releases](https://maven-badges.herokuapp.com/maven-central/com.twitter/twitter-server_2.12) are done on an approximately monthly schedule. While [semver](http://semver.org/) -is not followed, the [changelogs](CHANGES) are detailed and include sections on +is not followed, the [changelogs](CHANGELOG.rst) are detailed and include sections on public API breaks and changes in runtime behavior. ## Getting involved From f14d67c223001f93785255dfc65da500e0d6d6a0 Mon Sep 17 00:00:00 2001 From: Kevin Oliver Date: Mon, 27 Aug 2018 18:21:54 +0000 Subject: [PATCH 010/409] Dedent CHANGELOG.rst to remove blockquotes Problem The CHANGELOGs display with block quotes which is not intended. Solution Dedent the bullet points. Also, add the changelogs to user guides where missing. Result Nicer looking changelogs. JIRA Issues: CSL-6799 Differential Revision: https://phabricator.twitter.biz/D207427 --- CHANGELOG.rst | 236 +++++++++++++++++------------------ doc/src/sphinx/Changelog.rst | 7 ++ doc/src/sphinx/index.rst | 1 + 3 files changed, 126 insertions(+), 118 deletions(-) create mode 100644 doc/src/sphinx/Changelog.rst diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3e75a5cb..be04f1f5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,11 +10,11 @@ Unreleased Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ - * Move logic to parse the server `build.properties` file out the `c.t.server.handler.ServerInfoHandler` - and into a utility object, `c.t.server.BuildProperties` to allow for accessing by other server - logic such that the properties do not need to be re-parsed anytime access is desired. Failure to - load the properties can result in the server not starting in the case of a Fatal exception - being thrown. ``PHAB_ID=D201207`` +* Move logic to parse the server `build.properties` file out the `c.t.server.handler.ServerInfoHandler` + and into a utility object, `c.t.server.BuildProperties` to allow for accessing by other server + logic such that the properties do not need to be re-parsed anytime access is desired. Failure to + load the properties can result in the server not starting in the case of a Fatal exception + being thrown. ``PHAB_ID=D201207`` 18.8.0 ------- @@ -22,10 +22,10 @@ Runtime Behavior Changes New Features ~~~~~~~~~~~~ - * Add `onExit` lifecycle callback to `c.t.server.Hook` (which is now an abstract class) to allow - implemented hooks to execute functions in the `App#onExit` lifecycle phase. Note: - `c.t.server.Hook#premain` now has a default implementation and requires the `override` modifier. - ``PHAB_ID=D198379`` +* Add `onExit` lifecycle callback to `c.t.server.Hook` (which is now an abstract class) to allow + implemented hooks to execute functions in the `App#onExit` lifecycle phase. Note: + `c.t.server.Hook#premain` now has a default implementation and requires the `override` modifier. + ``PHAB_ID=D198379`` 18.7.0 ------- @@ -38,8 +38,8 @@ No Changes New Features ~~~~~~~~~~~~ - * Added an admin page at "/admin/balancers.json" with details about client load balancers, - including both configuration and current status. ``PHAB_ID=D171589`` +* Added an admin page at "/admin/balancers.json" with details about client load balancers, + including both configuration and current status. ``PHAB_ID=D171589`` 18.5.0 ------- @@ -47,8 +47,8 @@ New Features Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ - * Overloaded `c.t.server.AdminHttpServer#isolate` to accept a - `Service[Request, Response]`. ``PHAB_ID=D157891`` +* Overloaded `c.t.server.AdminHttpServer#isolate` to accept a + `Service[Request, Response]`. ``PHAB_ID=D157891`` 18.4.0 ------- @@ -66,8 +66,8 @@ No Changes Dependencies ~~~~~~~~~~~~ - * Removed 'finagle-zipkin-core' as a depdendency since there was no - code in twitter-server which used it. ``PHAB_ID=D129515`` +* Removed 'finagle-zipkin-core' as a depdendency since there was no + code in twitter-server which used it. ``PHAB_ID=D129515`` 18.1.0 ------- @@ -80,9 +80,9 @@ No Changes Bug Fixes ~~~~~~~~~ - * Treat `io.netty.channel.epoll.Native.epollWait0` as an idle thread on - "/admin/threads". This method is observed when using Netty 4's native - transport. ``PHAB_ID=D115058`` +* Treat `io.netty.channel.epoll.Native.epollWait0` as an idle thread on + "/admin/threads". This method is observed when using Netty 4's native + transport. ``PHAB_ID=D115058`` 17.11.0 ------- @@ -90,12 +90,12 @@ Bug Fixes Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ - * Change to apply JUL log format in the `c.t.server.logging.Logging` trait - constructor instead of in `premain` to apply format as early in the logging - stack as possible. However, this means that users overriding the - `def defaultFormatter` will not be able to use any flags to configure their - formatting, note: the default `LogFormatter` does not use flags. - ``PHAB_ID=D106534`` +* Change to apply JUL log format in the `c.t.server.logging.Logging` trait + constructor instead of in `premain` to apply format as early in the logging + stack as possible. However, this means that users overriding the + `def defaultFormatter` will not be able to use any flags to configure their + formatting, note: the default `LogFormatter` does not use flags. + ``PHAB_ID=D106534`` 17.10.0 ------- @@ -103,20 +103,20 @@ Breaking API Changes Release Version Format ~~~~~~~~~~~~~~~~~~~~~~ - * From now on, release versions will be based on release date in the format of - YY.MM.x where x is a patch number. ``PHAB_ID=D101244`` +* From now on, release versions will be based on release date in the format of + YY.MM.x where x is a patch number. ``PHAB_ID=D101244`` Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ - * All admin endpoints except ping + healthcheck are now by-default served outside - the global worker pool. ``PHAB_ID=D96633`` +* All admin endpoints except ping + healthcheck are now by-default served outside + the global worker pool. ``PHAB_ID=D96633`` Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ - * Rename AdminHttpServer#defaultHttpPort to AdminHttpServer#defaultAdminPort. - ``PHAB_ID=D97394`` +* Rename AdminHttpServer#defaultHttpPort to AdminHttpServer#defaultAdminPort. + ``PHAB_ID=D97394`` 1.32.0 ------ @@ -124,9 +124,9 @@ Breaking API Changes Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ - * Removed code related to `util-events` including `EventSink`, `JsonSink`, - `TraceEventSink`. The corresponding "/admin/events" and "/admin/events/record/" - admin HTTP endpoints are also removed. ``PHAB_ID=D82346`` +* Removed code related to `util-events` including `EventSink`, `JsonSink`, + `TraceEventSink`. The corresponding "/admin/events" and "/admin/events/record/" + admin HTTP endpoints are also removed. ``PHAB_ID=D82346`` 1.31.0 ------ @@ -138,14 +138,14 @@ No Changes Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ - * The admin server now waits for other registered closables to shut down - before commencing its own shutdown. ``RB_ID=916421`` +* The admin server now waits for other registered closables to shut down + before commencing its own shutdown. ``RB_ID=916421`` Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ - * Removed `c.t.server.Closer` trait. Behavior has been moved to - `c.t.app.App`. ``RB_ID=915485`` +* Removed `c.t.server.Closer` trait. Behavior has been moved to + `c.t.app.App`. ``RB_ID=915485`` 1.29.0 ------ @@ -158,7 +158,7 @@ No Changes Dependencies ~~~~~~~~~~~~ - * Bump guava to 19.0. ``RB_ID=907807`` +* Bump guava to 19.0. ``RB_ID=907807`` 1.27.0 ------ @@ -166,7 +166,7 @@ Dependencies New Features ~~~~~~~~~~~~ - * Add lint error warning on admin summary page. ``RB_ID=898202`` +* Add lint error warning on admin summary page. ``RB_ID=898202`` 1.26.0 ------ @@ -174,8 +174,8 @@ New Features Bug Fixes ~~~~~~~~~ - * Server graphs are now displaying again on the Twitter Server Summary page. - ``RB_ID=898422`` +* Server graphs are now displaying again on the Twitter Server Summary page. + ``RB_ID=898422`` 1.25.0 ------ @@ -183,13 +183,13 @@ Bug Fixes New Features ~~~~~~~~~~~~ - * Add ability to specify admin interface UI grouping, alias, and path for admin - handlers using the newly added Route and RouteUi. ``RB_ID=886829`` +* Add ability to specify admin interface UI grouping, alias, and path for admin + handlers using the newly added Route and RouteUi. ``RB_ID=886829`` Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ - * Removed `scala-xml` dependency. ``RB_ID=890315`` +* Removed `scala-xml` dependency. ``RB_ID=890315`` 1.24.0 ------ @@ -207,14 +207,14 @@ No Changes New Features ~~~~~~~~~~~~ - * No longer need to add an additional resolver that points to maven.twttr.com. - ``RB_ID=878967`` +* No longer need to add an additional resolver that points to maven.twttr.com. + ``RB_ID=878967`` Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ - * ShutdownHandler and AbortHandler accept only POST requests and ignore - non-POST requests. ``RB_ID=848212`` +* ShutdownHandler and AbortHandler accept only POST requests and ignore + non-POST requests. ``RB_ID=848212`` 1.21.0 ------ @@ -222,29 +222,29 @@ Runtime Behavior Changes Bug Fixes ~~~~~~~~~ - * Escape user input that is rendered in HTML, and make bin/travisci publish - finagle-toggle. ``RB_ID=848579`` +* Escape user input that is rendered in HTML, and make bin/travisci publish + finagle-toggle. ``RB_ID=848579`` New Features ~~~~~~~~~~~~ - * Add optional HTTP request parameter `filter` to `/admin/registry.json` - allowing for simple filtering of the returned JSON. ``RB_ID=842784`` +* Add optional HTTP request parameter `filter` to `/admin/registry.json` + allowing for simple filtering of the returned JSON. ``RB_ID=842784`` - * Add admin endpoint, `/admin/toggles`, for - `c.t.finagle.toggle.StandardToggleMap` registered `Toggles`. - ``RB_ID=847434`` +* Add admin endpoint, `/admin/toggles`, for + `c.t.finagle.toggle.StandardToggleMap` registered `Toggles`. + ``RB_ID=847434`` Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ - * Removed AdminHttpServer#mkRoutex method. This method was created during the - migration away from direct usage of netty http types and is now - redundant. ``RB_ID=835083`` +* Removed AdminHttpServer#mkRoutex method. This method was created during the + migration away from direct usage of netty http types and is now + redundant. ``RB_ID=835083`` - * Builds are now only for Java 8 and Scala 2.11. See the - `blog post `_ - for details. ``RB_ID=828898`` +* Builds are now only for Java 8 and Scala 2.11. See the + `blog post `_ + for details. ``RB_ID=828898`` 1.20.0 ------ @@ -252,15 +252,15 @@ Breaking API Changes Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ - * Introduce a new lifecycle event `prebindWarmup` for warmup code - which needs to run before the service handles traffic. GC has - moved from `warmupComplete` to `prebindWarmup`. ``RB_ID=819411`` +* Introduce a new lifecycle event `prebindWarmup` for warmup code + which needs to run before the service handles traffic. GC has + moved from `warmupComplete` to `prebindWarmup`. ``RB_ID=819411`` New Features ~~~~~~~~~~~~ - * Update to register TwitterServer as library in /admin/registry.json. ``RB_ID=825129`` - * Add a FailFast lint rule for Memcached client. ``RB_ID=808727`` +* Update to register TwitterServer as library in /admin/registry.json. ``RB_ID=825129`` +* Add a FailFast lint rule for Memcached client. ``RB_ID=808727`` 1.19.0 ------ @@ -268,8 +268,8 @@ New Features New Features ~~~~~~~~~~~~ - * Add AdminHttpServer#boundAddress to expose the bound address of - the AdminHttpServer. ``RB_ID=798322`` +* Add AdminHttpServer#boundAddress to expose the bound address of + the AdminHttpServer. ``RB_ID=798322`` 1.18.0 ------ @@ -277,7 +277,7 @@ New Features New Features ~~~~~~~~~~~~ - * Add new admin endpoint "/" which redirects requests to "/admin". ``RB_ID=777247`` +* Add new admin endpoint "/" which redirects requests to "/admin". ``RB_ID=777247`` 1.17.0 ------ @@ -293,14 +293,14 @@ NOT RELEASED New Features ~~~~~~~~~~~~ - * Add new admin endpoint "/admin/lint" which checks for possible issues with - performance or configuration. ``RB_ID=754348`` +* Add new admin endpoint "/admin/lint" which checks for possible issues with + performance or configuration. ``RB_ID=754348`` Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ - * We no longer export a "scheduler/productivity" stat because various implementation - details made it difficult to report reliably. +* We no longer export a "scheduler/productivity" stat because various implementation + details made it difficult to report reliably. 1.14.0 ------ @@ -308,9 +308,9 @@ Runtime Behavior Changes Dependencies ~~~~~~~~~~~~ - * Converted to finagle-httpx. Projects that depend transitively on - finagle-http through twitter-server will need to switch to finagle-httpx. - RB_ID=741454 RB_ID=740731 +* Converted to finagle-httpx. Projects that depend transitively on + finagle-http through twitter-server will need to switch to finagle-httpx. + ``RB_ID=741454`` ``RB_ID=740731`` 1.13.0 ------ @@ -318,7 +318,7 @@ Dependencies 1.12.0 ------ - * Enable syntax highlighting in the docs +* Enable syntax highlighting in the docs 1.11.0 ------ @@ -326,7 +326,7 @@ Dependencies New Features ~~~~~~~~~~~~ - * Introduce AbstractTwitterServer, a Java-friendly version of TwitterServer. RB_ID=661878 +* Introduce AbstractTwitterServer, a Java-friendly version of TwitterServer. RB_ID=661878 1.10.0 ------ @@ -334,17 +334,17 @@ New Features New Features ~~~~~~~~~~~~ - * TwitterServer collects a statically allocated ring of runtime events, which be viewed - at /admin/events, and downloaded as JSON with a HTTP client like curl (or by simply - omitting the User-Agent header in the request). +* TwitterServer collects a statically allocated ring of runtime events, which be viewed + at /admin/events, and downloaded as JSON with a HTTP client like curl (or by simply + omitting the User-Agent header in the request). - * TwitterServer exports runtime configuration data about your service, which can be - downloaded as json at /admin/registry.json. +* TwitterServer exports runtime configuration data about your service, which can be + downloaded as json at /admin/registry.json. Dependencies ~~~~~~~~~~~~ - * Bumped many dependency versions. +* Bumped many dependency versions. 1.9.0 ----- @@ -352,32 +352,32 @@ Dependencies New Features ~~~~~~~~~~~~ - * Most noticeably, the admin server received a complete UI redesign. All http endpoints are - now available via a navigation pane. We provide a simple API for service owners to include - ad-hoc admin pages which will be part of the navigation pane. Note, it is still possible to - join the admin server’s namespace via the global com.twitter.finagle.http.HttpMuxer. +* Most noticeably, the admin server received a complete UI redesign. All http endpoints are + now available via a navigation pane. We provide a simple API for service owners to include + ad-hoc admin pages which will be part of the navigation pane. Note, it is still possible to + join the admin server’s namespace via the global com.twitter.finagle.http.HttpMuxer. - * When using the twitter Metrics library, the admin server now scrapes your stats to - extract a quick summary of how your server is performing. This includes secondly - success rates for your server and least performant downstreams. We provide a /admin/metrics - endpoint which can watch stats (at secondly granularity) and extract them via http queries. +* When using the twitter Metrics library, the admin server now scrapes your stats to + extract a quick summary of how your server is performing. This includes secondly + success rates for your server and least performant downstreams. We provide a /admin/metrics + endpoint which can watch stats (at secondly granularity) and extract them via http queries. - * More recently, Finagle clients and servers began to retain information about their composition. - This is useful in examining the modules and parameters that comprise a specific implementation. - We now surface this information in the admin server via /admin/clients/ and - /admin/servers/ +* More recently, Finagle clients and servers began to retain information about their composition. + This is useful in examining the modules and parameters that comprise a specific implementation. + We now surface this information in the admin server via /admin/clients/ and + /admin/servers/ - * TwitterServer now exposes a /admin/registry.json endpoint, which speaks json and exposes the - values from util-registry as labels. Most labels are long-lived, and tend to represent something - about a process that is true for the entire lifetime, like the version of a library, or what a - flag was set to. +* TwitterServer now exposes a /admin/registry.json endpoint, which speaks json and exposes the + values from util-registry as labels. Most labels are long-lived, and tend to represent something + about a process that is true for the entire lifetime, like the version of a library, or what a + flag was set to. Dependencies ~~~~~~~~~~~~ - * Remove dependency on mustache for admin server in favor of templating - via string interpolation. This is more hygienic for web applications - and frameworks built atop twitter-server. +* Remove dependency on mustache for admin server in favor of templating + via string interpolation. This is more hygienic for web applications + and frameworks built atop twitter-server. 1.8.0 ----- @@ -385,33 +385,33 @@ Dependencies New Features ~~~~~~~~~~~~ - * Add the ability to promote objects to old gen before serving - * Export everything from build.properties at /admin/server_info +* Add the ability to promote objects to old gen before serving +* Export everything from build.properties at /admin/server_info Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ - * Add merge_base merge_base_commit_date and scm_repository to server_info - * AdminHttpServer now disables tracing - * Export gauge on eden allocations - * Improve heuristic for returning html or not in WebHandler - * Initial redesign of admin pages +* Add merge_base merge_base_commit_date and scm_repository to server_info +* AdminHttpServer now disables tracing +* Export gauge on eden allocations +* Improve heuristic for returning html or not in WebHandler +* Initial redesign of admin pages Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ - * Remove ServerInfo class: export /admin/server_info directly from build.properties file +* Remove ServerInfo class: export /admin/server_info directly from build.properties file 1.7.6 ----- -twitter-server: Add gauge on eden allocations -twitter-server: Do not trace the admin http server -twitter-server: JvmStats needs to call Allocations.start() -twitter-server: Log severely if a flag is read at the wrong time -twitter-server: Parameterize IndexHandler on a `patterns: Seq[String]` -twitter-server: Proper resource loading in admin pages -twitter-server: Redesign of twitter-server admin page +* twitter-server: Add gauge on eden allocations +* twitter-server: Do not trace the admin http server +* twitter-server: JvmStats needs to call Allocations.start() +* twitter-server: Log severely if a flag is read at the wrong time +* twitter-server: Parameterize IndexHandler on a `patterns: Seq[String]` +* twitter-server: Proper resource loading in admin pages +* twitter-server: Redesign of twitter-server admin page 1.7.3 ----- diff --git a/doc/src/sphinx/Changelog.rst b/doc/src/sphinx/Changelog.rst new file mode 100644 index 00000000..7bfe1440 --- /dev/null +++ b/doc/src/sphinx/Changelog.rst @@ -0,0 +1,7 @@ +Changelog +========= + +Below is a list of changes for each TwitterServer release. + + +.. include:: ../../../CHANGELOG.rst diff --git a/doc/src/sphinx/index.rst b/doc/src/sphinx/index.rst index 1bad4b04..4c940dd8 100644 --- a/doc/src/sphinx/index.rst +++ b/doc/src/sphinx/index.rst @@ -75,3 +75,4 @@ User’s guide Java Migration FAQ + Changelog From 72399a8ddff218a63e4a274d9e28ba919ec64140 Mon Sep 17 00:00:00 2001 From: Kevin Oliver Date: Mon, 27 Aug 2018 18:23:59 +0000 Subject: [PATCH 011/409] Add symlinks to CHANGELOG.rst from CHANGES Problem / Solution CHANGES got moved to CHANGELOG.rst. By leaving a symlink we keep existing posts and whatnot working. JIRA Issues: CSL-6799 Differential Revision: https://phabricator.twitter.biz/D207399 --- CHANGES | 1 + 1 file changed, 1 insertion(+) create mode 120000 CHANGES diff --git a/CHANGES b/CHANGES new file mode 120000 index 00000000..3824c8c1 --- /dev/null +++ b/CHANGES @@ -0,0 +1 @@ +CHANGELOG.rst \ No newline at end of file From 7d3ba30ed1cdb7dad347f30ef2a9ab061d02d4b1 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Wed, 29 Aug 2018 17:49:20 +0000 Subject: [PATCH 012/409] jackson: Upgrade from 2.8.4 to 2.9.6 Problem We're on a version of jackson from October 2016, and it has a [CVE][0] against it. Furthermore, there are bugfixes and feature improvements in new jackson, which we would like to use. Solution Upgrade to the latest and greatest version of jackson. [0]: https://nvd.nist.gov/vuln/detail/CVE-2017-7525 JIRA Issues: CSL-6599 Differential Revision: https://phabricator.twitter.biz/D204133 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 0cbfb2d9..208f4d62 100644 --- a/build.sbt +++ b/build.sbt @@ -3,7 +3,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch val releaseVersion = "18.9.0-SNAPSHOT" -val jacksonVersion = "2.8.4" +val jacksonVersion = "2.9.6" val jacksonLibs = Seq( "com.fasterxml.jackson.core" % "jackson-core" % jacksonVersion, "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion, From 544ac16c5a69ee6f25c63f388c7b4247036cd305 Mon Sep 17 00:00:00 2001 From: Automated go/ktf Flagger Date: Fri, 7 Sep 2018 01:53:18 +0000 Subject: [PATCH 013/409] The test target "twitter-server/server/src/test/scala:scala" is failing too often in master in CI. Summary: Problem: The test target "twitter-server/server/src/test/scala:scala" is failing too often in master in CI. Logs available here: Known to fail ticket: [CSL-6954](https://jira.twitter.biz/browse/CSL-6954) Solution: It has been marked "known_to_fail" in the BUILD file Result: This will cause it to be skipped in future builds, and its failures ignored in submit queue. This implies that code changes that further break this test will not be prevented from being submitted to the master branch, until the problem is fixed and the test is no longer marked known-to-fail. TBR=true Reviewers: O5268 source:/twitter-server/! Subscribers: #rb_csl Differential Revision: https://phabricator.twitter.biz/D212094 --- server/src/test/scala/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/test/scala/BUILD b/server/src/test/scala/BUILD index a58bf965..72f0ae66 100644 --- a/server/src/test/scala/BUILD +++ b/server/src/test/scala/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = rglobs("*.scala"), fatal_warnings = True, + known_to_fail = "CSL-6954", dependencies = [ "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", "3rdparty/jvm/com/fasterxml/jackson/module:jackson-module-scala", From 944b7294eea62769cc53e30f6ecab91a243fe444 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Mon, 10 Sep 2018 19:51:11 +0000 Subject: [PATCH 014/409] twitter-server: Update ShutdownTimer to use Finagle DefaultTimer Problem `TwitterServer` uses the inherited `ShutdownTimer` from `c.t.app.App` which is the standard `c.t.util.JavaTimer` when it should instead probably prefer the Finagle `DefaultTimer`. Solution Update `TwitterServer` trait to override the inherited `ShutdownTimer` to be the Finagle `DefaultTimer` instead of the `c.t.util.JavaTimer` defined by default in `c.t.app.App`. JIRA Issues: CSL-6954 Differential Revision: https://phabricator.twitter.biz/D212896 --- CHANGELOG.rst | 5 +++++ .../src/main/scala/com/twitter/server/TwitterServer.scala | 8 ++++++-- server/src/test/scala/BUILD | 1 - server/src/test/scala/com/twitter/server/HookTest.scala | 4 ++++ .../test/scala/com/twitter/server/TwitterServerTest.scala | 4 +++- 5 files changed, 18 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index be04f1f5..b62e0862 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -16,6 +16,11 @@ Runtime Behavior Changes load the properties can result in the server not starting in the case of a Fatal exception being thrown. ``PHAB_ID=D201207`` +* Update `TwitterServer` trait to override the inherited `ShutdownTimer` to be the Finagle + `DefaultTimer` instead of the `c.t.util.JavaTimer` defined by default in `c.t.app.App`. Also + update the overridden `suppressGracefulShutdownErrors` in `TwitterServer` to be a val since + it is constant (instead of a def). ``PHAB_ID=D212896`` + 18.8.0 ------- diff --git a/server/src/main/scala/com/twitter/server/TwitterServer.scala b/server/src/main/scala/com/twitter/server/TwitterServer.scala index aa97b4b3..494f6726 100644 --- a/server/src/main/scala/com/twitter/server/TwitterServer.scala +++ b/server/src/main/scala/com/twitter/server/TwitterServer.scala @@ -1,6 +1,8 @@ package com.twitter.server import com.twitter.app.App +import com.twitter.finagle.util.DefaultTimer +import com.twitter.util.Timer import com.twitter.util.logging.Logging /** @@ -40,8 +42,10 @@ trait TwitterServer with Lifecycle with Stats { - // don't let applications opt-out - final override def suppressGracefulShutdownErrors: Boolean = false + /** Don't let applications opt-out */ + final override val suppressGracefulShutdownErrors: Boolean = false + /** Use the Finagle DefaultTimer */ + override protected lazy val shutdownTimer: Timer = DefaultTimer } /** diff --git a/server/src/test/scala/BUILD b/server/src/test/scala/BUILD index 72f0ae66..a58bf965 100644 --- a/server/src/test/scala/BUILD +++ b/server/src/test/scala/BUILD @@ -1,7 +1,6 @@ junit_tests( sources = rglobs("*.scala"), fatal_warnings = True, - known_to_fail = "CSL-6954", dependencies = [ "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", "3rdparty/jvm/com/fasterxml/jackson/module:jackson-module-scala", diff --git a/server/src/test/scala/com/twitter/server/HookTest.scala b/server/src/test/scala/com/twitter/server/HookTest.scala index 86c6c1af..c03bf5ba 100644 --- a/server/src/test/scala/com/twitter/server/HookTest.scala +++ b/server/src/test/scala/com/twitter/server/HookTest.scala @@ -1,5 +1,7 @@ package com.twitter.server +import com.twitter.conversions.time._ +import com.twitter.util.Duration import org.scalatest.FunSuite import scala.collection.mutable @@ -17,6 +19,8 @@ class OnExitHook2 extends Hook { class HookTestTwitterServer(hooks: Seq[Hook]) extends TwitterServer { override val defaultAdminPort = 0 + /* ensure enough time to close resources */ + override val defaultCloseGracePeriod: Duration = 30.seconds val bootstrapSeq: mutable.MutableList[Symbol] = mutable.MutableList.empty[Symbol] diff --git a/server/src/test/scala/com/twitter/server/TwitterServerTest.scala b/server/src/test/scala/com/twitter/server/TwitterServerTest.scala index 50dfc82b..9695c75b 100644 --- a/server/src/test/scala/com/twitter/server/TwitterServerTest.scala +++ b/server/src/test/scala/com/twitter/server/TwitterServerTest.scala @@ -1,5 +1,6 @@ package com.twitter.server +import com.twitter.conversions.time._ import com.twitter.finagle.Service import com.twitter.finagle.http.{Request, Response} import com.twitter.util._ @@ -10,6 +11,8 @@ import scala.collection.mutable class TestTwitterServer extends TwitterServer { override val defaultAdminPort = 0 + /* ensure enough time to close resources */ + override val defaultCloseGracePeriod: Duration = 30.seconds val bootstrapSeq: mutable.MutableList[Symbol] = mutable.MutableList.empty[Symbol] @@ -42,7 +45,6 @@ class MockExceptionHandler extends Service[Request, Response] { } @RunWith(classOf[JUnitRunner]) class TwitterServerTest extends FunSuite { - test("TwitterServer does not prematurely execute lifecycle hooks") { val twitterServer = new TestTwitterServer assert(twitterServer.bootstrapSeq.isEmpty) From 32043403d11de4726087b22d62087b57ce3fa81c Mon Sep 17 00:00:00 2001 From: Neuman Vong Date: Tue, 11 Sep 2018 18:47:41 +0000 Subject: [PATCH 015/409] Twitter-oss: Prepare OSS libraries for release 18.9 Problem We want to release the next versions of our Twitter OSS libraries 18.9 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-6957 Differential Revision: https://phabricator.twitter.biz/D213029 --- CHANGELOG.rst | 3 +++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b62e0862..40a9c4a8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +18.9.0 +------- + Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/README.md b/README.md index 9df46cb3..ba72642e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 208f4d62..d812c4d6 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "18.9.0-SNAPSHOT" +val releaseVersion = "18.9.0" val jacksonVersion = "2.9.6" val jacksonLibs = Seq( From 72079978ae6a94947a1caeb5935a7ca14ad3f96a Mon Sep 17 00:00:00 2001 From: Danny McClanahan Date: Wed, 12 Sep 2018 00:40:07 +0000 Subject: [PATCH 016/409] Convert deprecated fatal_warnings usages to use the new compiler_option_sets from pants 1.9.x! Problem `fatal_warnings` is now deprecated in favor of the more extensible `compiler_option_sets` pants provides in 1.9.x. Solution Convert all `fatal_warnings` arguments to use the new 1.9.x `compiler_option_sets` in all `pants.ini.*`, `BUILD`, and other files. This was done with a combination of `rg` and `sed`: ``` rg -l 'fatal_warnings\s*=\s*.?True' \ | parallel -L50 "sed -r -i -e 's#fatal_warnings(\s*=\s*)\"?True\"?\s*,#compiler_option_sets\1{\"fatal_warnings\"},#g'" \ && rg -l 'fatal_warnings\s*=\s*.?False' \ | parallel -L50 "sed -r -i -e 's#fatal_warnings(\s*=\s*)\"?False\"?\s*,#compiler_option_sets\1\{\},#g'" \ && rg -l 'fatal_warnings.*=' ``` Result Fixes deprecation warnings when running pants on jvm code. JIRA Issues: DPB-10250 TBR=true NO_USER_HOOK=1 Differential Revision: https://phabricator.twitter.biz/D213208 --- logback-classic/src/main/scala/BUILD | 42 +++++++------- logback-classic/src/test/scala/BUILD | 22 +++---- server/src/main/scala/BUILD | 86 ++++++++++++++-------------- server/src/test/java/BUILD | 4 +- server/src/test/scala/BUILD | 4 +- slf4j-jdk14/src/main/scala/BUILD | 42 +++++++------- slf4j-jdk14/src/test/scala/BUILD | 24 ++++---- slf4j-log4j12/src/main/scala/BUILD | 46 +++++++-------- slf4j-log4j12/src/test/scala/BUILD | 22 +++---- 9 files changed, 146 insertions(+), 146 deletions(-) diff --git a/logback-classic/src/main/scala/BUILD b/logback-classic/src/main/scala/BUILD index 6b613b7a..1e664307 100644 --- a/logback-classic/src/main/scala/BUILD +++ b/logback-classic/src/main/scala/BUILD @@ -1,27 +1,27 @@ scala_library( + sources = rglobs("*.scala"), + compiler_option_sets = {"fatal_warnings"}, provides = scala_artifact( - org='com.twitter', - name='twitter-server-logback-classic', - repo=artifactory, + org = "com.twitter", + name = "twitter-server-logback-classic", + repo = artifactory, ), - dependencies=[ - '3rdparty/jvm/org/slf4j:slf4j-api', - '3rdparty/jvm/ch/qos/logback:logback-classic', - '3rdparty/jvm/ch/qos/logback:logback-core', - '3rdparty/jvm/org/slf4j:jcl-over-slf4j', - '3rdparty/jvm/org/slf4j:log4j-over-slf4j', - '3rdparty/jvm/org/slf4j:jul-to-slf4j', - 'finagle/finagle-base-http/src/main/scala', - 'finagle/finagle-http/src/main/scala', - 'twitter-server/logback-classic/src/main/resources', - 'twitter-server/server/src/main/scala', - 'util/util-slf4j-api/src/main/scala', + scope = "forced", # Exposes a service-loaded class. + dependencies = [ + "3rdparty/jvm/ch/qos/logback:logback-classic", + "3rdparty/jvm/ch/qos/logback:logback-core", + "3rdparty/jvm/org/slf4j:jcl-over-slf4j", + "3rdparty/jvm/org/slf4j:jul-to-slf4j", + "3rdparty/jvm/org/slf4j:log4j-over-slf4j", + "3rdparty/jvm/org/slf4j:slf4j-api", + "finagle/finagle-base-http/src/main/scala", + "finagle/finagle-http/src/main/scala", + "twitter-server/logback-classic/src/main/resources", + "twitter-server/server/src/main/scala", + "util/util-slf4j-api/src/main/scala", ], - fatal_warnings=True, - sources=rglobs('*.scala'), - scope='forced', # Exposes a service-loaded class. - exports=[ - '3rdparty/jvm/org/slf4j:slf4j-api', - '3rdparty/jvm/ch/qos/logback:logback-classic', + exports = [ + "3rdparty/jvm/ch/qos/logback:logback-classic", + "3rdparty/jvm/org/slf4j:slf4j-api", ], ) diff --git a/logback-classic/src/test/scala/BUILD b/logback-classic/src/test/scala/BUILD index 9f8e9e5c..4ec8347c 100644 --- a/logback-classic/src/test/scala/BUILD +++ b/logback-classic/src/test/scala/BUILD @@ -1,14 +1,14 @@ junit_tests( - dependencies=[ - '3rdparty/jvm/ch/qos/logback:logback-classic', - '3rdparty/jvm/junit', - '3rdparty/jvm/org/scalatest:scalatest', - '3rdparty/jvm/org/slf4j:slf4j-api', - 'finagle/finagle-core/src/main/scala:scala', - 'twitter-server/logback-classic/src/main/scala:scala', - 'twitter-server/server/src/main/scala:scala', - 'util/util-slf4j-api/src/main/scala', + sources = rglobs("*.scala"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/ch/qos/logback:logback-classic", + "3rdparty/jvm/junit", + "3rdparty/jvm/org/scalatest", + "3rdparty/jvm/org/slf4j:slf4j-api", + "finagle/finagle-core/src/main/scala", + "twitter-server/logback-classic/src/main/scala", + "twitter-server/server/src/main/scala", + "util/util-slf4j-api/src/main/scala", ], - fatal_warnings=True, - sources=rglobs('*.scala'), ) diff --git a/server/src/main/scala/BUILD b/server/src/main/scala/BUILD index 84e8d942..a0b6d9a5 100644 --- a/server/src/main/scala/BUILD +++ b/server/src/main/scala/BUILD @@ -1,47 +1,47 @@ scala_library( - provides=scala_artifact( - org='com.twitter', - name='twitter-server', - repo=artifactory, + sources = rglobs("*.scala"), + compiler_option_sets = {"fatal_warnings"}, + provides = scala_artifact( + org = "com.twitter", + name = "twitter-server", + repo = artifactory, ), - dependencies=[ - '3rdparty/jvm/com/fasterxml/jackson/core:jackson-annotations', - '3rdparty/jvm/com/fasterxml/jackson/core:jackson-core', - '3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind', - '3rdparty/jvm/com/fasterxml/jackson/module:jackson-module-scala', - '3rdparty/jvm/io/netty:netty', - '3rdparty/jvm/io/netty:netty4', - '3rdparty/jvm/org/slf4j:slf4j-api', - 'finagle/finagle-base-http/src/main/scala', - 'finagle/finagle-core/src/main/scala', - 'finagle/finagle-http/src/main/scala', - 'finagle/finagle-netty4-http/src/main/scala', - 'finagle/finagle-netty4/src/main/scala', - 'finagle/finagle-toggle/src/main/scala', - 'finagle/finagle-tunable/src/main/scala', - 'twitter-server/server/src/main/resources', - 'util/util-app/src/main/scala', - 'util/util-core/src/main/scala', - 'util/util-jvm/src/main/scala', - 'util/util-lint/src/main/scala', - 'util/util-registry/src/main/scala', - 'util/util-stats/src/main/scala', - 'util/util-slf4j-api/src/main/scala', - 'util/util-slf4j-jul-bridge/src/main/scala', - 'util/util-tunable/src/main/scala', + scope = "forced", # Exposes a service-loaded class. + dependencies = [ + "3rdparty/jvm/com/fasterxml/jackson/core:jackson-annotations", + "3rdparty/jvm/com/fasterxml/jackson/core:jackson-core", + "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", + "3rdparty/jvm/com/fasterxml/jackson/module:jackson-module-scala", + "3rdparty/jvm/io/netty", + "3rdparty/jvm/io/netty:netty4", + "3rdparty/jvm/org/slf4j:slf4j-api", + "finagle/finagle-base-http/src/main/scala", + "finagle/finagle-core/src/main/scala", + "finagle/finagle-http/src/main/scala", + "finagle/finagle-netty4-http/src/main/scala", + "finagle/finagle-netty4/src/main/scala", + "finagle/finagle-toggle/src/main/scala", + "finagle/finagle-tunable/src/main/scala", + "twitter-server/server/src/main/resources", + "util/util-app/src/main/scala", + "util/util-core/src/main/scala", + "util/util-jvm/src/main/scala", + "util/util-lint/src/main/scala", + "util/util-registry/src/main/scala", + "util/util-slf4j-api/src/main/scala", + "util/util-slf4j-jul-bridge/src/main/scala", + "util/util-stats/src/main/scala", + "util/util-tunable/src/main/scala", + ], + exports = [ + "3rdparty/jvm/com/fasterxml/jackson/core:jackson-annotations", + "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", + "3rdparty/jvm/org/slf4j:slf4j-api", + "finagle/finagle-base-http/src/main/scala", + "finagle/finagle-http/src/main/scala", + "util/util-app/src/main/scala", + "util/util-core/src/main/scala", + "util/util-lint/src/main/scala", + "util/util-slf4j-api/src/main/scala", ], - fatal_warnings=True, - sources=rglobs('*.scala'), - scope='forced', # Exposes a service-loaded class. - exports=[ - '3rdparty/jvm/com/fasterxml/jackson/core:jackson-annotations', - '3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind', - '3rdparty/jvm/org/slf4j:slf4j-api', - 'finagle/finagle-base-http/src/main/scala', - 'finagle/finagle-http/src/main/scala', - 'util/util-app/src/main/scala', - 'util/util-core/src/main/scala', - 'util/util-lint/src/main/scala', - 'util/util-slf4j-api/src/main/scala', - ] ) diff --git a/server/src/test/java/BUILD b/server/src/test/java/BUILD index 90f366d2..f61eb507 100644 --- a/server/src/test/java/BUILD +++ b/server/src/test/java/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = rglobs("*.java"), - fatal_warnings = False, + compiler_option_sets = {}, dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", @@ -17,4 +17,4 @@ junit_tests( "util/util-slf4j-api/src/main/scala", "util/util-stats/src/main/scala", ], -) \ No newline at end of file +) diff --git a/server/src/test/scala/BUILD b/server/src/test/scala/BUILD index a58bf965..39a99a4e 100644 --- a/server/src/test/scala/BUILD +++ b/server/src/test/scala/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = rglobs("*.scala"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", "3rdparty/jvm/com/fasterxml/jackson/module:jackson-module-scala", @@ -23,4 +23,4 @@ junit_tests( "util/util-stats/src/main/scala", "util/util-tunable/src/main/scala", ], -) \ No newline at end of file +) diff --git a/slf4j-jdk14/src/main/scala/BUILD b/slf4j-jdk14/src/main/scala/BUILD index f3c47dea..c9dadbf8 100644 --- a/slf4j-jdk14/src/main/scala/BUILD +++ b/slf4j-jdk14/src/main/scala/BUILD @@ -1,26 +1,26 @@ scala_library( - provides=scala_artifact( - org='com.twitter', - name='twitter-server-slf4j-jdk14', - repo=artifactory, + sources = rglobs("*.scala"), + compiler_option_sets = {"fatal_warnings"}, + provides = scala_artifact( + org = "com.twitter", + name = "twitter-server-slf4j-jdk14", + repo = artifactory, ), - dependencies=[ - '3rdparty/jvm/org/slf4j:slf4j-api', - '3rdparty/jvm/org/slf4j:jcl-over-slf4j', - '3rdparty/jvm/org/slf4j:log4j-over-slf4j', - '3rdparty/jvm/org/slf4j:slf4j-jdk14', - 'finagle/finagle-base-http/src/main/scala', - 'finagle/finagle-http/src/main/scala', - 'twitter-server/server/src/main/scala', - 'twitter-server/slf4j-jdk14/src/main/resources', - 'util/util-logging/src/main/scala', + scope = "forced", # Exposes a service-loaded class. + dependencies = [ + "3rdparty/jvm/org/slf4j:jcl-over-slf4j", + "3rdparty/jvm/org/slf4j:log4j-over-slf4j", + "3rdparty/jvm/org/slf4j:slf4j-api", + "3rdparty/jvm/org/slf4j:slf4j-jdk14", + "finagle/finagle-base-http/src/main/scala", + "finagle/finagle-http/src/main/scala", + "twitter-server/server/src/main/scala", + "twitter-server/slf4j-jdk14/src/main/resources", + "util/util-logging/src/main/scala", ], - fatal_warnings=True, - sources=rglobs('*.scala'), - scope='forced', # Exposes a service-loaded class. - exports=[ - '3rdparty/jvm/org/slf4j:slf4j-api', - '3rdparty/jvm/org/slf4j:slf4j-jdk14', - 'util/util-logging/src/main/scala', + exports = [ + "3rdparty/jvm/org/slf4j:slf4j-api", + "3rdparty/jvm/org/slf4j:slf4j-jdk14", + "util/util-logging/src/main/scala", ], ) diff --git a/slf4j-jdk14/src/test/scala/BUILD b/slf4j-jdk14/src/test/scala/BUILD index 01177c16..6c615854 100644 --- a/slf4j-jdk14/src/test/scala/BUILD +++ b/slf4j-jdk14/src/test/scala/BUILD @@ -1,15 +1,15 @@ junit_tests( - dependencies=[ - '3rdparty/jvm/junit', - '3rdparty/jvm/org/mockito:mockito-all', - '3rdparty/jvm/org/scalacheck', - '3rdparty/jvm/org/scalatest', - 'finagle/finagle-base-http/src/main/scala', - 'finagle/finagle-core/src/main/scala', - 'finagle/finagle-http/src/main/scala', - 'twitter-server/server/src/main/scala:scala', - 'twitter-server/slf4j-jdk14/src/main/scala:scala', + sources = rglobs("*.scala"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/junit", + "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/scalacheck", + "3rdparty/jvm/org/scalatest", + "finagle/finagle-base-http/src/main/scala", + "finagle/finagle-core/src/main/scala", + "finagle/finagle-http/src/main/scala", + "twitter-server/server/src/main/scala", + "twitter-server/slf4j-jdk14/src/main/scala", ], - fatal_warnings=True, - sources=rglobs('*.scala') ) diff --git a/slf4j-log4j12/src/main/scala/BUILD b/slf4j-log4j12/src/main/scala/BUILD index 3b8f3d2a..f575c444 100644 --- a/slf4j-log4j12/src/main/scala/BUILD +++ b/slf4j-log4j12/src/main/scala/BUILD @@ -1,27 +1,27 @@ scala_library( - provides=scala_artifact( - org='com.twitter', - name='twitter-server-slf4j-log4j12', - repo=artifactory, + sources = rglobs("*.scala"), + compiler_option_sets = {"fatal_warnings"}, + provides = scala_artifact( + org = "com.twitter", + name = "twitter-server-slf4j-log4j12", + repo = artifactory, ), - dependencies=[ - '3rdparty/jvm/log4j', - '3rdparty/jvm/org/slf4j:slf4j-api', - '3rdparty/jvm/org/slf4j:jcl-over-slf4j', - '3rdparty/jvm/org/slf4j:jul-to-slf4j', - '3rdparty/jvm/org/slf4j:slf4j-log4j12', - 'finagle/finagle-base-http/src/main/scala', - 'finagle/finagle-http/src/main/scala', - 'twitter-server/server/src/main/scala', - 'twitter-server/slf4j-log4j12/src/main/resources', - 'util/util-slf4j-api/src/main/scala', + scope = "forced", # Exposes a service-loaded class. + dependencies = [ + "3rdparty/jvm/log4j", + "3rdparty/jvm/org/slf4j:jcl-over-slf4j", + "3rdparty/jvm/org/slf4j:jul-to-slf4j", + "3rdparty/jvm/org/slf4j:slf4j-api", + "3rdparty/jvm/org/slf4j:slf4j-log4j12", + "finagle/finagle-base-http/src/main/scala", + "finagle/finagle-http/src/main/scala", + "twitter-server/server/src/main/scala", + "twitter-server/slf4j-log4j12/src/main/resources", + "util/util-slf4j-api/src/main/scala", + ], + exports = [ + "3rdparty/jvm/log4j", + "3rdparty/jvm/org/slf4j:slf4j-api", + "3rdparty/jvm/org/slf4j:slf4j-log4j12", ], - fatal_warnings=True, - sources=rglobs('*.scala'), - scope='forced', # Exposes a service-loaded class. - exports=[ - '3rdparty/jvm/log4j', - '3rdparty/jvm/org/slf4j:slf4j-api', - '3rdparty/jvm/org/slf4j:slf4j-log4j12', - ] ) diff --git a/slf4j-log4j12/src/test/scala/BUILD b/slf4j-log4j12/src/test/scala/BUILD index ca3d4ee5..d733a26c 100644 --- a/slf4j-log4j12/src/test/scala/BUILD +++ b/slf4j-log4j12/src/test/scala/BUILD @@ -1,14 +1,14 @@ junit_tests( - dependencies=[ - '3rdparty/jvm/junit', - '3rdparty/jvm/log4j:log4j', - '3rdparty/jvm/org/scalatest:scalatest', - '3rdparty/jvm/org/slf4j:slf4j-api', - 'finagle/finagle-core/src/main/scala:scala', - 'twitter-server/server/src/main/scala:scala', - 'twitter-server/slf4j-log4j12/src/main/scala:scala', - 'util/util-slf4j-api/src/main/scala', + sources = rglobs("*.scala"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/junit", + "3rdparty/jvm/log4j", + "3rdparty/jvm/org/scalatest", + "3rdparty/jvm/org/slf4j:slf4j-api", + "finagle/finagle-core/src/main/scala", + "twitter-server/server/src/main/scala", + "twitter-server/slf4j-log4j12/src/main/scala", + "util/util-slf4j-api/src/main/scala", ], - fatal_warnings=True, - sources=rglobs('*.scala'), ) From e9c5baaa1a9e6ddda32dc7c624d30a1e596ca59e Mon Sep 17 00:00:00 2001 From: Neuman Vong Date: Mon, 17 Sep 2018 16:53:59 +0000 Subject: [PATCH 017/409] Twitter-oss: update OSS libraries post-release to 18.10.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 18.10.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D214858 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ba72642e..9df46cb3 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index d812c4d6..ab2d1c3c 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "18.9.0" +val releaseVersion = "18.10.0-SNAPSHOT" val jacksonVersion = "2.9.6" val jacksonLibs = Seq( From 83459231ab1a3ab3d73f247b0539f343cb9d80af Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Wed, 19 Sep 2018 20:44:38 +0000 Subject: [PATCH 018/409] twitter-oss: Update to Scala 2.11.12 and 2.12.6 in SBT build Problem/Solution We should update our SBT builds to Scala 2.11.12 and 2.12.6. Additionally, the Slack credentials for each project are out-of-date and need to be updated. JIRA Issues: CSL-6972, CSL-6975 Differential Revision: https://phabricator.twitter.biz/D216930 --- .travis.yml | 6 +++--- build.sbt | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index f2c5f102..b84edfca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,15 +19,15 @@ before_cache: - find $HOME/.sbt -name "*.lock" -delete scala: - - 2.11.11 - - 2.12.4 + - 2.11.12 + - 2.12.6 jdk: - oraclejdk8 notifications: slack: - secure: HFhFrzUWBycBX5NO+wqBSvqBrFsK+Tj2qJMj0baqiIZEFSDpnNf+Lif4uXxdJG9Y03/66pKdDMMCE3dM4OOyKWh6vXq8lEWk7g9yVBxAud0UZ84DL1Y1iRToK0+I3SRcL6tjwsVAvTiuYyPXwnYQ8f/AqTs0wqEvNI/T9DZRX4E= + secure: FfzYHh8IvtBaYswh9mIQU9yQGr9YpkRaFR77RaYdK5wVhw3UlCBtRTBdFpUn0djgg3Pl3FxEj5OyEKYQ803xxF7UQRxx298g4r5OhOzjQ4UH1lhnAg1WpIRPBL4DuWrreZFtnqvsygrdtvATZ1aLzaYtkaCV49x/t2Kr7weNfpQ= before_script: # default $SBT_OPTS is irrelevant to sbt lancher diff --git a/build.sbt b/build.sbt index ab2d1c3c..fa90170a 100644 --- a/build.sbt +++ b/build.sbt @@ -25,8 +25,8 @@ lazy val noPublishSettings = Seq( lazy val sharedSettings = Seq( version := releaseVersion, organization := "com.twitter", - scalaVersion := "2.12.4", - crossScalaVersions := Seq("2.11.11", "2.12.4"), + scalaVersion := "2.12.6", + crossScalaVersions := Seq("2.11.12", "2.12.6"), libraryDependencies ++= Seq( "org.scalacheck" %% "scalacheck" % "1.13.4" % "test", "org.scalatest" %% "scalatest" % "3.0.0" % "test", From 6f934acbc506abd59eb93faaba01f7d0b4276ece Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Thu, 20 Sep 2018 23:43:29 +0000 Subject: [PATCH 019/409] twitter-oss: Update TravisCI to use sudo = required Problem It has been recommended by the TravisCI support that we switch our builds to use `sudo: required` in order to get a fully virtualized environment for running our builds. See: https://docs.travis-ci.com/user/reference/trusty/#fully-virtualized-via-sudo-required We hadn't switched as we were beholden to needing a Java install which wasn't readily available previously but is available in the TravisCI "trusty" image. Solution Switch the TravisCI configuration to use `sudo: required` and `dist: trusty`. Differential Revision: https://phabricator.twitter.biz/D218377 --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b84edfca..b38e56cc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ -# container-based build -sudo: false +dist: trusty +sudo: required language: scala From 0eaa63608d41de786837df66bfde00aa945d1175 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Mon, 24 Sep 2018 17:59:36 +0000 Subject: [PATCH 020/409] twitter-oss: Introduce Dodo build stage in TravisCI config Problem Currently, running the dodo script to build the dependencies of a project happens *before every job* when it only needs to run only once before all jobs. This causes unnecessary computation and churn. Additionally, we are running "./sbt update" unnecessarily as it was done because of the flakiness with `maven.twttr.com` in resolving dependencies which would kill builds. Since we no longer rely on it, we can remove the manual resolution of dependencies in the jobs. Solution Introduce a Dodo build stage and order it before the default test stage such that it runs before any testing and only once. The output is cached, which is then used in the subsequent jobs in the test build stage. This has the additional benefit of removing the time spent building with dodo from the test jobs, which means they have more time to finish within the TravisCI job timeout. Result Better stability with our TravisCI testing. Building with Dodo is only run once for the TravisCI build, it has up to the default time for a job to finish, and every test afterwards also now can spend up to the maximum timeout to actually run tests. JIRA Issues: CSL-6974 Differential Revision: https://phabricator.twitter.biz/D219136 --- .travis.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index b38e56cc..f596d8df 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,14 +30,20 @@ notifications: secure: FfzYHh8IvtBaYswh9mIQU9yQGr9YpkRaFR77RaYdK5wVhw3UlCBtRTBdFpUn0djgg3Pl3FxEj5OyEKYQ803xxF7UQRxx298g4r5OhOzjQ4UH1lhnAg1WpIRPBL4DuWrreZFtnqvsygrdtvATZ1aLzaYtkaCV49x/t2Kr7weNfpQ= before_script: - # default $SBT_OPTS is irrelevant to sbt lancher - - unset SBT_OPTS - - curl -s https://raw.githubusercontent.com/twitter/dodo/develop/bin/build | bash -s -- --no-test twitter-server - - travis_retry ./sbt --error ++$TRAVIS_SCALA_VERSION update + - unset SBT_OPTS # default $SBT_OPTS is irrelevant to sbt launcher script: - travis_retry ./sbt ++$TRAVIS_SCALA_VERSION clean coverage test coverageReport - - ./sbt ++$TRAVIS_SCALA_VERSION coverageAggregate after_success: + - ./sbt ++$TRAVIS_SCALA_VERSION coverageAggregate - bash <(curl -s https://codecov.io/bash) + +stages: + - dodo + - test + +jobs: + include: + - stage: dodo + script: curl -s https://raw.githubusercontent.com/twitter/dodo/develop/bin/build | bash -s -- --no-test twitter-server From 9a0cd842320b28785b41365e2592bcdefaf72f0a Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Wed, 26 Sep 2018 20:14:03 +0000 Subject: [PATCH 021/409] twitter-oss: Prepare OSS libraries for release 18.9.1 Problem We want to release the next versions of our Twitter OSS libraries 18.9.1 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D220746 --- CHANGELOG.rst | 5 +++++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 40a9c4a8..9aac81f7 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +18.9.1 +------- + +No Changes + 18.9.0 ------- diff --git a/README.md b/README.md index 9df46cb3..ba72642e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index fa90170a..f93f6048 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "18.10.0-SNAPSHOT" +val releaseVersion = "18.9.1" val jacksonVersion = "2.9.6" val jacksonLibs = Seq( From 278aeba43e3a9d49fbd2f89b21f729457b0d37c9 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Thu, 27 Sep 2018 17:44:41 +0000 Subject: [PATCH 022/409] twitter-oss: Update OSS Libraries Post-Release Back to 18.10.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 18.10.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D221275 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ba72642e..9df46cb3 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index f93f6048..fa90170a 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "18.9.1" +val releaseVersion = "18.10.0-SNAPSHOT" val jacksonVersion = "2.9.6" val jacksonLibs = Seq( From b0dc398170b912f72fed7f73efb3313a371e7e6a Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Fri, 28 Sep 2018 00:29:12 +0000 Subject: [PATCH 023/409] TravisCI: Turn down the Slack notifications Problem The Slack notification on TravisCI builds is fairly noisy. Solution Change to only notify on change of either status. TBR=true Differential Revision: https://phabricator.twitter.biz/D221661 --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index f596d8df..9a4378d1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,6 +28,8 @@ jdk: notifications: slack: secure: FfzYHh8IvtBaYswh9mIQU9yQGr9YpkRaFR77RaYdK5wVhw3UlCBtRTBdFpUn0djgg3Pl3FxEj5OyEKYQ803xxF7UQRxx298g4r5OhOzjQ4UH1lhnAg1WpIRPBL4DuWrreZFtnqvsygrdtvATZ1aLzaYtkaCV49x/t2Kr7weNfpQ= + on_failure: change + on_success: change before_script: - unset SBT_OPTS # default $SBT_OPTS is irrelevant to sbt launcher From a84996279f3426ca74c5770676e45814e38bf1b8 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Sat, 29 Sep 2018 20:06:29 +0000 Subject: [PATCH 024/409] TravisCI: Remove Build Stages configuration Problem Time is a flat circle. Everything we've ever done or will do, we're going to do over and over again. Build stages rely on the caching of artifacts to S3 in order fro jobs to share state. While this seems like a great idea in theory. The practice has proven to be somewhat untenable. Solution Walk away from build stages. This really only affects the Finagle build as it is the only build which remains a matrix build and thus the only build which potentially runs the dodo step multiple times. Given the complexity around trying to get build stages to work and the cache to be properly created and shared for different scala versions, this is an acceptable amount of overhead. JIRA Issues: CSL-7054 TBR=true Differential Revision: https://phabricator.twitter.biz/D222286 --- .travis.yml | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9a4378d1..3150ff95 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,20 +3,25 @@ sudo: required language: scala +git: + depth: 3 + quiet: true + env: - JAVA_OPTS="-DSKIP_FLAKY=true -Dsbt.log.noformat=true" # These directories are cached to S3 at the end of the build cache: directories: - - $HOME/.ivy2 - - $HOME/.sbt/boot/scala-$TRAVIS_SCALA_VERSION + - $HOME/.ivy2/cache + - $HOME/.ivy2/local/com.twitter + - $HOME/.sbt - $HOME/.dodo before_cache: - # Tricks to avoid unnecessary cache updates - - find $HOME/.ivy2 -name "ivydata-*.properties" -delete - - find $HOME/.sbt -name "*.lock" -delete + # Cleanup the cached directories to avoid unnecessary cache updates + - find $HOME/.ivy2/cache -name "ivydata-*.properties" -delete + - find $HOME/.sbt -name "*.lock" -delete scala: - 2.11.12 @@ -33,6 +38,7 @@ notifications: before_script: - unset SBT_OPTS # default $SBT_OPTS is irrelevant to sbt launcher + - curl -s https://raw.githubusercontent.com/twitter/dodo/develop/bin/build | bash -s -- --no-test --scala-version $TRAVIS_SCALA_VERSION twitter-server script: - travis_retry ./sbt ++$TRAVIS_SCALA_VERSION clean coverage test coverageReport @@ -40,12 +46,3 @@ script: after_success: - ./sbt ++$TRAVIS_SCALA_VERSION coverageAggregate - bash <(curl -s https://codecov.io/bash) - -stages: - - dodo - - test - -jobs: - include: - - stage: dodo - script: curl -s https://raw.githubusercontent.com/twitter/dodo/develop/bin/build | bash -s -- --no-test twitter-server From a4447361bd588b89bcd0749706b8bc01fec6dc8f Mon Sep 17 00:00:00 2001 From: David Rusek Date: Tue, 2 Oct 2018 21:39:18 +0000 Subject: [PATCH 025/409] Whitespace change to force github sync Problem/Solution: Twitter Server did not sync cleanly sometime in the past. Make a trivial (whitespace) change to force the sync to occur and fail so we can manually merge. JIRA Issues: CSL-7063 Differential Revision: https://phabricator.twitter.biz/D223262 --- logback-classic/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/logback-classic/README.md b/logback-classic/README.md index d9a6689e..c9c1e3fb 100644 --- a/logback-classic/README.md +++ b/logback-classic/README.md @@ -13,7 +13,7 @@ Users MUST provide a compatible version of the [Logback](https://logback.qos.ch/ Admin HTTP Interface Logging Handler ------------------------------------ -Depending on this library will install a logging handler on the [HTTP admin interface](https://twitter.github.io/Admin.html#admin-interface) +Depending on this library will install a logging handler on the [HTTP admin interface](https://twitter.github.io/twitter-server/Admin.html#admin-interface) which allows users to dynamically change `ch.qos.logback.classic.Logger` log levels. -For more information see the `TwitterServer` user guide section on [Logging](https://twitter.github.io/Features.html#logging) +For more information see the `TwitterServer` user guide section on [Logging](https://twitter.github.io/twitter-server/Features.html#logging) From 971194a0b0f34b4d5964271d9926770368fd55d5 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Mon, 15 Oct 2018 23:17:16 +0000 Subject: [PATCH 026/409] Upgrade scalac 2.12 to 2.12.7 Problem / Solution There is a new scalac 2.12.x version so lets upgrade. JIRA Issues: CSL-7111 Differential Revision: https://phabricator.twitter.biz/D228382 --- .travis.yml | 2 +- build.sbt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3150ff95..e88de433 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,7 @@ before_cache: scala: - 2.11.12 - - 2.12.6 + - 2.12.7 jdk: - oraclejdk8 diff --git a/build.sbt b/build.sbt index fa90170a..2e085151 100644 --- a/build.sbt +++ b/build.sbt @@ -25,8 +25,8 @@ lazy val noPublishSettings = Seq( lazy val sharedSettings = Seq( version := releaseVersion, organization := "com.twitter", - scalaVersion := "2.12.6", - crossScalaVersions := Seq("2.11.12", "2.12.6"), + scalaVersion := "2.12.7", + crossScalaVersions := Seq("2.11.12", "2.12.7"), libraryDependencies ++= Seq( "org.scalacheck" %% "scalacheck" % "1.13.4" % "test", "org.scalatest" %% "scalatest" % "3.0.0" % "test", From 7ce6e4cbcaf3ac9d9dccd62a5aaf9c9572cf5788 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Wed, 17 Oct 2018 17:56:44 +0000 Subject: [PATCH 027/409] twitter-server: Do not warn when there is no build.properties to load Problem/Solution The `c.t.server.BuildProperties` is rather noisy when there is no `build.properties` to load, i.e., when running integration tests that start a server. This results in a `NullPointerException`, so we specifically change to handle that case in loading the properties file to not output a log message. Thus the warning should appear only when there is a problem loading an actual file. Differential Revision: https://phabricator.twitter.biz/D229586 --- CHANGELOG.rst | 26 ++++++++++++------- .../com/twitter/server/BuildProperties.scala | 7 ++++- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9aac81f7..6da706ff 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,13 +7,19 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Runtime Behavior Changes +~~~~~~~~~~~~~~~~~~~~~~~~ + +* Update `BuildProperties` to not emit a warning when no `build.properties` file can be + located. ``PHAB_ID=D229586`` + 18.9.1 -------- +------ No Changes 18.9.0 -------- +------ Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -30,7 +36,7 @@ Runtime Behavior Changes it is constant (instead of a def). ``PHAB_ID=D212896`` 18.8.0 -------- +------ New Features ~~~~~~~~~~~~ @@ -41,12 +47,12 @@ New Features ``PHAB_ID=D198379`` 18.7.0 -------- +------ No Changes 18.6.0 -------- +------ New Features ~~~~~~~~~~~~ @@ -55,7 +61,7 @@ New Features including both configuration and current status. ``PHAB_ID=D171589`` 18.5.0 -------- +------ Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -64,17 +70,17 @@ Runtime Behavior Changes `Service[Request, Response]`. ``PHAB_ID=D157891`` 18.4.0 -------- +------ No Changes 18.3.0 -------- +------ No Changes 18.2.0 -------- +------ Dependencies ~~~~~~~~~~~~ @@ -83,7 +89,7 @@ Dependencies code in twitter-server which used it. ``PHAB_ID=D129515`` 18.1.0 -------- +------ No Changes diff --git a/server/src/main/scala/com/twitter/server/BuildProperties.scala b/server/src/main/scala/com/twitter/server/BuildProperties.scala index b2fcda4f..1241f6ca 100644 --- a/server/src/main/scala/com/twitter/server/BuildProperties.scala +++ b/server/src/main/scala/com/twitter/server/BuildProperties.scala @@ -1,6 +1,7 @@ package com.twitter.server import com.twitter.util.logging.Logger +import java.net.URL import java.util.Properties import scala.collection.JavaConverters._ import scala.util.control.NonFatal @@ -32,7 +33,11 @@ private[server] object BuildProperties { } catch { case NonFatal(_) => try { - buildProperties.load(BuildProperties.getClass.getResource("/build.properties").openStream) + BuildProperties.getClass.getResource("/build.properties") match { + case resource: URL => + buildProperties.load(resource.openStream) + case _ => // do nothing + } } catch { case NonFatal(e) => log.warn("Unable to load build.properties file from classpath. " + e.getMessage) From ef20f6b4b60f2a65e7445e7ff6a09bfbdc8b1a2e Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Thu, 18 Oct 2018 00:15:26 +0000 Subject: [PATCH 028/409] twitter-server: Update OWNERS file Problem / Solution I can do more reviews if I'm an owner of twitter-server, so make myself an owner. Differential Revision: https://phabricator.twitter.biz/D230110 --- OWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OWNERS b/OWNERS index 63c2ce64..3b2291e3 100644 --- a/OWNERS +++ b/OWNERS @@ -1,6 +1,6 @@ +banderson ccoco dschobel -imartin jparker koliver mnakamura From 4078e4cb39812f1085bd38fdcaca2de35f2dffc7 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Thu, 18 Oct 2018 19:09:06 +0000 Subject: [PATCH 029/409] Deprecate c.t.server.AdminHttpServer#routes Problem `c.t.server.AdminHttpServer#routes` should have been marked deprecated some time ago. Solution Mark `c.t.server.AdminHttpServer#routes` as deprecated. Differential Revision: https://phabricator.twitter.biz/D230247 --- CHANGELOG.rst | 6 ++++++ .../src/main/scala/com/twitter/server/AdminHttpServer.scala | 3 +-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6da706ff..337a9137 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,12 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Changes +~~~~~~~ + +* Deprecate `c.t.server.AdminHttpServer#routes`. Routes should be added to the `AdminHttpServer` + via `c.t.server.AdminHttpServer#addAdminRoutes`. ``PHAB_ID=D230247`` + Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/server/src/main/scala/com/twitter/server/AdminHttpServer.scala b/server/src/main/scala/com/twitter/server/AdminHttpServer.scala index 1ab6def9..51d4353a 100644 --- a/server/src/main/scala/com/twitter/server/AdminHttpServer.scala +++ b/server/src/main/scala/com/twitter/server/AdminHttpServer.scala @@ -180,8 +180,7 @@ trait AdminHttpServer { self: App => addAdminRoutes(Seq(route)) } - // TODO: remove routes, have all routes be added via addAdminRoutes - // For now these routes will be added to allRoutes in premain + @deprecated("Routes should be added via `AdminHttpServer#addAdminRoutes`", "2018-10-17") protected def routes: Seq[Route] = Nil /** From c725b8ab2c05f7bd095d83751379d4d7a47c0b03 Mon Sep 17 00:00:00 2001 From: Jordan Parker Date: Thu, 18 Oct 2018 23:28:58 +0000 Subject: [PATCH 030/409] Twitter-oss: Prepare OSS libraries for release 18.10.0 Problem We want to release the next versions of our Twitter OSS libraries 18.10.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-7105 Differential Revision: https://phabricator.twitter.biz/D230664 --- CHANGELOG.rst | 3 +++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 337a9137..3dd53ae5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +18.10.0 +------- + Changes ~~~~~~~ diff --git a/README.md b/README.md index 9df46cb3..ba72642e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 2e085151..01036924 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "18.10.0-SNAPSHOT" +val releaseVersion = "18.10.0" val jacksonVersion = "2.9.6" val jacksonLibs = Seq( From 2045a08263fc387e27d8424cdb2b47fee39059cc Mon Sep 17 00:00:00 2001 From: Jordan Parker Date: Fri, 19 Oct 2018 20:40:48 +0000 Subject: [PATCH 031/409] Twitter-oss: update OSS libraries post-release to 18.11.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. TBR=true Differential Revision: https://phabricator.twitter.biz/D231095 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ba72642e..9df46cb3 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 01036924..dddff5a2 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "18.10.0" +val releaseVersion = "18.11.0-SNAPSHOT" val jacksonVersion = "2.9.6" val jacksonLibs = Seq( From 5cb29295ef7494b80ce81f8a3f338735f886d2bd Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Thu, 1 Nov 2018 02:10:13 +0000 Subject: [PATCH 032/409] twitter-server: Enable scalafmt # Problem / Solution We're enabling scalafmt internally. This commit does it for Twitter Server. JIRA Issues: CSL-7225 Differential Revision: https://phabricator.twitter.biz/D236250 --- .../logback/classic/LoggingHandler.scala | 102 +++++++++--------- .../logback/classic/LoggingHandlerTest.scala | 2 +- .../com/twitter/server/AdminHttpServer.scala | 21 ++-- .../main/scala/com/twitter/server/Stats.scala | 1 + .../com/twitter/server/TwitterServer.scala | 19 ++-- .../server/handler/ShutdownHandler.scala | 13 ++- .../lint/DuplicateLoadServiceBindings.scala | 15 +-- .../twitter/server/lint/LoggingRules.scala | 3 +- .../handler/slf4j/jdk14/LoggingHandler.scala | 4 +- .../slf4j/jdk14/AbstractTwitterServer.scala | 5 +- .../com/twitter/server/HandlersTest.scala | 1 - .../slf4j/log4j12/LoggingHandler.scala | 8 +- 12 files changed, 104 insertions(+), 90 deletions(-) diff --git a/logback-classic/src/main/scala/com/twitter/server/handler/logback/classic/LoggingHandler.scala b/logback-classic/src/main/scala/com/twitter/server/handler/logback/classic/LoggingHandler.scala index a7996b97..ed9f05d9 100644 --- a/logback-classic/src/main/scala/com/twitter/server/handler/logback/classic/LoggingHandler.scala +++ b/logback-classic/src/main/scala/com/twitter/server/handler/logback/classic/LoggingHandler.scala @@ -39,13 +39,12 @@ private class LoggingHandler extends AdminHttpMuxHandler with Logging { javalog.Level.FINE, javalog.Level.FINER, javalog.Level.FINEST, - javalog.Level.ALL) - .sorted(julLevelOrder) + javalog.Level.ALL + ).sorted(julLevelOrder) /** Exposed for testing */ private[classic] def loggers = - LoggerFactory - .getILoggerFactory + LoggerFactory.getILoggerFactory .asInstanceOf[LoggerContext] .getLoggerList .asScala @@ -151,44 +150,44 @@ private class LoggingHandler extends AdminHttpMuxHandler with Logging { ${(for (logger <- renderLoggers) yield { - val loggerName = - if (logger.getName == "") org.slf4j.Logger.ROOT_LOGGER_NAME else logger.getName - val inheritsLevel = logger.getLevel == null - val filterQueryParams = if (showOverriddenOnly) { - "?overridden=true&" - } else "?" + val loggerName = + if (logger.getName == "") org.slf4j.Logger.ROOT_LOGGER_NAME else logger.getName + val inheritsLevel = logger.getLevel == null + val filterQueryParams = if (showOverriddenOnly) { + "?overridden=true&" + } else "?" - val buttons = for (level <- levels) yield { - val isActive = logger.getEffectiveLevel == level + val buttons = for (level <- levels) yield { + val isActive = logger.getEffectiveLevel == level - val activeCss = if (!isActive) "btn-default" - else if (!inheritsLevel) "btn-primary active disabled" - else "btn-primary active" + val activeCss = + if (!isActive) "btn-default" + else if (!inheritsLevel) "btn-primary active disabled" + else "btn-primary active" - val queryParams = - if (isActive && logger.getLevel == logger.getEffectiveLevel) "" - else { - s"""logger=${URLEncoder.encode(loggerName, "UTF-8")}&level=${level.toString}""" - } + val queryParams = + if (isActive && logger.getLevel == logger.getEffectiveLevel) "" + else { + s"""logger=${URLEncoder.encode(loggerName, "UTF-8")}&level=${level.toString}""" + } - s"""${level.toString}""" - } + } - val resetButton = if (!inheritsLevel && loggerName != org.slf4j.Logger.ROOT_LOGGER_NAME) { - val queryParams = s"""logger=${URLEncoder.encode(loggerName, "UTF-8")}&level=null""" - s"""RESET""" - } else "" + val resetButton = if (!inheritsLevel && loggerName != org.slf4j.Logger.ROOT_LOGGER_NAME) { + val queryParams = s"""logger=${URLEncoder.encode(loggerName, "UTF-8")}&level=null""" + s"""RESET""" + } else "" - s""" + s"""
${escapeHtml(loggerName)}
${buttons.mkString("\n")} $resetButton
""" - }).mkString("\n") - } + }).mkString("\n")}

java.util.Logging Loggers

@@ -198,35 +197,38 @@ private class LoggingHandler extends AdminHttpMuxHandler with Logging { - ${val filterQueryParams = if (showOverriddenOnly) { - "?overridden=true&" - } else "?" + ${ + val filterQueryParams = if (showOverriddenOnly) { + "?overridden=true&" + } else "?" - (for (logger <- julLoggers) yield { - val loggerName = getLoggerDisplayName(logger) - val buttons = (for (level <- julLevels) yield { - val isActive = getLevel(logger) == level - val activeCss = - if (!isActive) "btn-default" - else { - "btn-primary active disabled" - } - val queryParams = - if (isActive) "" - else { - s"""logger=${URLEncoder.encode(loggerName, "UTF-8")}&level=${level.toString}&isJul=true""" - } - s"""${level.toString}""" - }).mkString("\n") + }).mkString("\n") - s""" + s"""""" - }).mkString("\n")} + }).mkString("\n") + }
java.util.logging.Level
${escapeHtml(loggerName)}
$buttons
""" } diff --git a/logback-classic/src/test/scala/com/twitter/server/handler/logback/classic/LoggingHandlerTest.scala b/logback-classic/src/test/scala/com/twitter/server/handler/logback/classic/LoggingHandlerTest.scala index 0ead325e..d9e24793 100644 --- a/logback-classic/src/test/scala/com/twitter/server/handler/logback/classic/LoggingHandlerTest.scala +++ b/logback-classic/src/test/scala/com/twitter/server/handler/logback/classic/LoggingHandlerTest.scala @@ -103,7 +103,7 @@ class LoggingHandlerTest extends FunSuite { assert(overriddenHtml2.contains("baz")) - log.setLevel(null) // log level is now inherited, no longer overridden + log.setLevel(null) // log level is now inherited, no longer overridden val overriddenHtml3 = Await.result(handler(req), 5.seconds).contentString assert(!overriddenHtml3.contains("baz")) diff --git a/server/src/main/scala/com/twitter/server/AdminHttpServer.scala b/server/src/main/scala/com/twitter/server/AdminHttpServer.scala index 51d4353a..98bf0b06 100644 --- a/server/src/main/scala/com/twitter/server/AdminHttpServer.scala +++ b/server/src/main/scala/com/twitter/server/AdminHttpServer.scala @@ -226,9 +226,11 @@ trait AdminHttpServer { self: App => addLoggingHandler() // ensure there is an /admin/logging handler // create a service which multiplexes across all endpoints. - val localMuxer = allRoutes.foldLeft(new HttpMuxer) { case (muxer, route) => - val service = new IndexView(route.alias, route.path, () => indexEntries).andThen(route.handler) - muxer.withHandler(route.path, service) + val localMuxer = allRoutes.foldLeft(new HttpMuxer) { + case (muxer, route) => + val service = + new IndexView(route.alias, route.path, () => indexEntries).andThen(route.handler) + muxer.withHandler(route.path, service) } val endpoints = allRoutes.map { route => @@ -271,12 +273,13 @@ trait AdminHttpServer { self: App => routes .filter(_.includeInIndex) .groupBy(_.group) - .flatMap { case (groupOpt, rts) => - val links = rts.map(routeToIndexLink) - groupOpt match { - case Some(group) => Seq(IndexView.Group(group, links)) - case None => links - } + .flatMap { + case (groupOpt, rts) => + val links = rts.map(routeToIndexLink) + groupOpt match { + case Some(group) => Seq(IndexView.Group(group, links)) + case None => links + } } .toSeq } diff --git a/server/src/main/scala/com/twitter/server/Stats.scala b/server/src/main/scala/com/twitter/server/Stats.scala index 3fa9eb59..79166f71 100644 --- a/server/src/main/scala/com/twitter/server/Stats.scala +++ b/server/src/main/scala/com/twitter/server/Stats.scala @@ -5,6 +5,7 @@ import com.twitter.finagle.stats.{StatsReceiver, LoadedStatsReceiver} import com.twitter.server.util._ trait Stats { app: App => + /** * By default this returns the global [[LoadedStatsReceiver]] instance. Take note when * overriding this method to return the *same instance* on multiple calls otherwise you diff --git a/server/src/main/scala/com/twitter/server/TwitterServer.scala b/server/src/main/scala/com/twitter/server/TwitterServer.scala index 494f6726..1c5e11ce 100644 --- a/server/src/main/scala/com/twitter/server/TwitterServer.scala +++ b/server/src/main/scala/com/twitter/server/TwitterServer.scala @@ -32,18 +32,19 @@ import com.twitter.util.logging.Logging * logging APIs. */ trait TwitterServer - extends App - with Slf4jBridge - with Logging - with Linters - with Hooks - with AdminHttpServer - with Admin - with Lifecycle - with Stats { + extends App + with Slf4jBridge + with Logging + with Linters + with Hooks + with AdminHttpServer + with Admin + with Lifecycle + with Stats { /** Don't let applications opt-out */ final override val suppressGracefulShutdownErrors: Boolean = false + /** Use the Finagle DefaultTimer */ override protected lazy val shutdownTimer: Timer = DefaultTimer } diff --git a/server/src/main/scala/com/twitter/server/handler/ShutdownHandler.scala b/server/src/main/scala/com/twitter/server/handler/ShutdownHandler.scala index 629cbb50..8134a857 100644 --- a/server/src/main/scala/com/twitter/server/handler/ShutdownHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/ShutdownHandler.scala @@ -21,8 +21,11 @@ class ShutdownHandler(app: App) extends Service[Request, Response] { catch { case e: NumberFormatException => val msg = s"could not parse 'grace' parameter: $d is not a valid duration" - log.info(s"[${req.uri}] from ${req.remoteAddress.getHostAddress} " + - s"failed to quit due to invalid grace period format", e) + log.info( + s"[${req.uri}] from ${req.remoteAddress.getHostAddress} " + + s"failed to quit due to invalid grace period format", + e + ) return newResponse( status = Status.BadRequest, contentType = "text/plain;charset=UTF-8", @@ -31,8 +34,10 @@ class ShutdownHandler(app: App) extends Service[Request, Response] { } } val grace = specifiedGracePeriod.getOrElse(app.defaultCloseGracePeriod) - log.info(s"[${req.uri}] from ${req.remoteAddress.getHostAddress} " + - s"quitting with grace period $grace") + log.info( + s"[${req.uri}] from ${req.remoteAddress.getHostAddress} " + + s"quitting with grace period $grace" + ) app.close(grace) newOk("quitting\n") } else { diff --git a/server/src/main/scala/com/twitter/server/lint/DuplicateLoadServiceBindings.scala b/server/src/main/scala/com/twitter/server/lint/DuplicateLoadServiceBindings.scala index 81cc3062..9c4b3b0d 100644 --- a/server/src/main/scala/com/twitter/server/lint/DuplicateLoadServiceBindings.scala +++ b/server/src/main/scala/com/twitter/server/lint/DuplicateLoadServiceBindings.scala @@ -8,18 +8,19 @@ import com.twitter.util.lint.{Category, Issue, Rule} */ object DuplicateLoadServiceBindings { - def apply(): Rule = Rule( - Category.Configuration, - "Duplicate calls to `LoadService.bind`", - """ + def apply(): Rule = + Rule( + Category.Configuration, + "Duplicate calls to `LoadService.bind`", + """ |`LoadService.bind` allows users to specify a specific |implementation. If this is getting called multiple |times for the same interface, it indicates a setup/configuration |issue that may cause surprises. """.stripMargin - ) { - issues(LoadService.duplicateBindings) - } + ) { + issues(LoadService.duplicateBindings) + } /** exposed for testing */ private[lint] def issues(dupes: Set[Class[_]]): Seq[Issue] = diff --git a/server/src/main/scala/com/twitter/server/lint/LoggingRules.scala b/server/src/main/scala/com/twitter/server/lint/LoggingRules.scala index b9ba921f..9616dfdd 100644 --- a/server/src/main/scala/com/twitter/server/lint/LoggingRules.scala +++ b/server/src/main/scala/com/twitter/server/lint/LoggingRules.scala @@ -46,7 +46,8 @@ object LoggingRules { "Admin logging handler implementation to dynamically change log levels is not configured", "To configure, please add a dependency on one of the supported TwitterServer logging " + "implementations which will provide the ability to dynamically change the logging levels " + - "for that implementation: logback-classic, slf4j-log4j12, or slf4j-jdk14.") { + "for that implementation: logback-classic, slf4j-log4j12, or slf4j-jdk14." + ) { Seq(Issue("No logging handler implementation configured.")) } } diff --git a/slf4j-jdk14/src/main/scala/com/twitter/server/handler/slf4j/jdk14/LoggingHandler.scala b/slf4j-jdk14/src/main/scala/com/twitter/server/handler/slf4j/jdk14/LoggingHandler.scala index 6069d301..5b902d77 100644 --- a/slf4j-jdk14/src/main/scala/com/twitter/server/handler/slf4j/jdk14/LoggingHandler.scala +++ b/slf4j-jdk14/src/main/scala/com/twitter/server/handler/slf4j/jdk14/LoggingHandler.scala @@ -137,7 +137,9 @@ private class LoggingHandler extends AdminHttpMuxHandler { } } - private[this] def parseLoggerName(params: scala.collection.Map[String, scala.collection.Seq[String]]): Option[String] = { + private[this] def parseLoggerName( + params: scala.collection.Map[String, scala.collection.Seq[String]] + ): Option[String] = { params.getOrElse("logger", Seq.empty).headOption.map { case "root" => "" case name => name diff --git a/slf4j-jdk14/src/main/scala/com/twitter/server/slf4j/jdk14/AbstractTwitterServer.scala b/slf4j-jdk14/src/main/scala/com/twitter/server/slf4j/jdk14/AbstractTwitterServer.scala index 42c791ae..94068eb3 100644 --- a/slf4j-jdk14/src/main/scala/com/twitter/server/slf4j/jdk14/AbstractTwitterServer.scala +++ b/slf4j-jdk14/src/main/scala/com/twitter/server/slf4j/jdk14/AbstractTwitterServer.scala @@ -3,13 +3,10 @@ package com.twitter.server.slf4j.jdk14 import com.twitter.server.logging.Logging import com.twitter.server.{AbstractTwitterServer => MainAbstractTwitterServer} - /** * A Java-friendly version of the [[com.twitter.server.TwitterServer]] which provides a * backwards-compatible JUL logging implementation. * * @see [[com.twitter.server.AbstractTwitterServer]] */ -abstract class AbstractTwitterServer - extends MainAbstractTwitterServer - with Logging +abstract class AbstractTwitterServer extends MainAbstractTwitterServer with Logging diff --git a/slf4j-jdk14/src/test/scala/com/twitter/server/HandlersTest.scala b/slf4j-jdk14/src/test/scala/com/twitter/server/HandlersTest.scala index dd41718b..819c5f23 100644 --- a/slf4j-jdk14/src/test/scala/com/twitter/server/HandlersTest.scala +++ b/slf4j-jdk14/src/test/scala/com/twitter/server/HandlersTest.scala @@ -39,7 +39,6 @@ class TestTwitterServer extends TwitterServer { } } - class MockExceptionHandler extends Service[Request, Response] { val pattern = "/exception_please.json" def apply(req: Request): Future[Response] = { diff --git a/slf4j-log4j12/src/main/scala/com/twitter/server/handler/slf4j/log4j12/LoggingHandler.scala b/slf4j-log4j12/src/main/scala/com/twitter/server/handler/slf4j/log4j12/LoggingHandler.scala index 1c10b2d4..311c9689 100644 --- a/slf4j-log4j12/src/main/scala/com/twitter/server/handler/slf4j/log4j12/LoggingHandler.scala +++ b/slf4j-log4j12/src/main/scala/com/twitter/server/handler/slf4j/log4j12/LoggingHandler.scala @@ -110,8 +110,9 @@ private class LoggingHandler extends AdminHttpMuxHandler with Logging { private def renderHtml( loggerList: Seq[Logger], - levels: Seq[Level], updateMsg: - String, showOverriddenOnly: Boolean + levels: Seq[Level], + updateMsg: String, + showOverriddenOnly: Boolean ): String = s""" @@ -128,7 +129,8 @@ private class LoggingHandler extends AdminHttpMuxHandler with Logging { val inheritsLevel = logger.getLevel == null val buttons = for (level <- levels) yield { val isActive = logger.getEffectiveLevel == level - val activeCss = if (!isActive) "btn-default" + val activeCss = + if (!isActive) "btn-default" else if (!inheritsLevel) "btn-primary active disabled" else "btn-primary active" val href = From 6b7c313849a8b35a45ccc950ca06732286bc4ed7 Mon Sep 17 00:00:00 2001 From: Kevin Oliver Date: Mon, 12 Nov 2018 18:46:00 +0000 Subject: [PATCH 033/409] Twitter-oss: Prepare OSS libraries for release 18.11 Problem We want to release the next versions of our Twitter OSS libraries 18.11 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-7204 Differential Revision: https://phabricator.twitter.biz/D240782 --- CHANGELOG.rst | 5 +++++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3dd53ae5..49609872 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +18.11.0 +------- + +No Changes + 18.10.0 ------- diff --git a/README.md b/README.md index 9df46cb3..ba72642e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index dddff5a2..0728a5aa 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "18.11.0-SNAPSHOT" +val releaseVersion = "18.11.0" val jacksonVersion = "2.9.6" val jacksonLibs = Seq( From 06a3f520204d6aeb9e347563ff4ab8121ea7b11f Mon Sep 17 00:00:00 2001 From: Kevin Oliver Date: Tue, 13 Nov 2018 16:34:08 +0000 Subject: [PATCH 034/409] Twitter-oss: update OSS libraries post-release to 18.12 Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D241161 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ba72642e..9df46cb3 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 0728a5aa..fb326ec4 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "18.11.0" +val releaseVersion = "18.12.0-SNAPSHOT" val jacksonVersion = "2.9.6" val jacksonLibs = Seq( From 2da58b9a562bf5c1b20f56cca0283e992125e075 Mon Sep 17 00:00:00 2001 From: Daniel Schobel Date: Tue, 4 Dec 2018 01:20:37 +0000 Subject: [PATCH 035/409] finagle/util/scrooge/twitter-server: update owners Problem / Solution Update OWNERS files. Differential Revision: https://phabricator.twitter.biz/D248571 --- OWNERS | 1 - 1 file changed, 1 deletion(-) diff --git a/OWNERS b/OWNERS index 3b2291e3..b28cdc37 100644 --- a/OWNERS +++ b/OWNERS @@ -1,6 +1,5 @@ banderson ccoco -dschobel jparker koliver mnakamura From 839cb3c16fb7396f2bd4dba3e1adf88dab79c474 Mon Sep 17 00:00:00 2001 From: Owen Parry Date: Wed, 5 Dec 2018 21:42:37 +0000 Subject: [PATCH 036/409] Update scalafmt config for twitter-server. Problem: The scalafmt config doesn't match the Twitter Style Guide. This causes angst and makes onboarding new folks onto scalafmt a difficult sell. Solution: Update the scalafmt config, and everyone who has opted into automatic application of scalafmt. Result: The code automatically matches the style guide, and people are happier to adopt scalafmt. --- doc/src/sphinx/code/StackServer.scala | 6 ++---- .../com/twitter/server/AdminHttpServer.scala | 9 +++------ .../scala/com/twitter/server/Lifecycle.scala | 3 +-- .../server/handler/ClientRegistryHandler.scala | 7 +++---- .../server/handler/HistogramQueryHandler.scala | 3 +-- .../server/handler/ProfileResourceHandler.scala | 4 +--- .../twitter/server/handler/RegistryHandler.scala | 4 +--- .../twitter/server/handler/ResourceHandler.scala | 16 ++++++---------- .../server/handler/ServerRegistryHandler.scala | 4 ++-- .../twitter/server/handler/SummaryHandler.scala | 5 +---- .../twitter/server/handler/ToggleHandler.scala | 13 +++---------- .../server/lint/SchedulerBlockingRule.scala | 5 +---- .../com/twitter/server/util/MetricSource.scala | 3 +-- .../server/view/EndpointRegistryView.scala | 4 +--- .../com/twitter/server/view/IndexView.scala | 5 +---- .../twitter/server/view/StackRegistryView.scala | 5 +---- 16 files changed, 29 insertions(+), 67 deletions(-) diff --git a/doc/src/sphinx/code/StackServer.scala b/doc/src/sphinx/code/StackServer.scala index baa16dcd..982e6df4 100644 --- a/doc/src/sphinx/code/StackServer.scala +++ b/doc/src/sphinx/code/StackServer.scala @@ -4,7 +4,5 @@ import com.twitter.server.{Lifecycle, Deciderable} //#imports //#server -abstract class StackServer extends AbstractTwitterServer - with Lifecycle.Warmup - with Deciderable -//#server \ No newline at end of file +abstract class StackServer extends AbstractTwitterServer with Lifecycle.Warmup with Deciderable +//#server diff --git a/server/src/main/scala/com/twitter/server/AdminHttpServer.scala b/server/src/main/scala/com/twitter/server/AdminHttpServer.scala index 98bf0b06..7d69fecf 100644 --- a/server/src/main/scala/com/twitter/server/AdminHttpServer.scala +++ b/server/src/main/scala/com/twitter/server/AdminHttpServer.scala @@ -54,8 +54,7 @@ object AdminHttpServer { alias: String, group: Option[String], includeInIndex: Boolean, - method: Method = Get - ) + method: Method = Get) object Route { @@ -65,10 +64,8 @@ object AdminHttpServer { */ private[this] val IsolateFilter: SimpleFilter[Request, Response] = new SimpleFilter[Request, Response] { - def apply( - request: Request, - service: Service[Request, Response] - ): Future[Response] = Pool(service(request)).flatten + def apply(request: Request, service: Service[Request, Response]): Future[Response] = + Pool(service(request)).flatten override def toString: String = s"${Route.getClass.getName}IsolateFilter" } diff --git a/server/src/main/scala/com/twitter/server/Lifecycle.scala b/server/src/main/scala/com/twitter/server/Lifecycle.scala index 156699a9..b7e879b8 100644 --- a/server/src/main/scala/com/twitter/server/Lifecycle.scala +++ b/server/src/main/scala/com/twitter/server/Lifecycle.scala @@ -46,8 +46,7 @@ object promoteBeforeServing object Lifecycle { private[server] class PromoteToOldGen( - runtimeArgs: Seq[String] = ManagementFactory.getRuntimeMXBean.getInputArguments.asScala - ) { + runtimeArgs: Seq[String] = ManagementFactory.getRuntimeMXBean.getInputArguments.asScala) { private[this] val hasPromoted = new AtomicBoolean(false) private[server] def promoted: Boolean = hasPromoted.get() diff --git a/server/src/main/scala/com/twitter/server/handler/ClientRegistryHandler.scala b/server/src/main/scala/com/twitter/server/handler/ClientRegistryHandler.scala index 7d59a222..16c72e54 100644 --- a/server/src/main/scala/com/twitter/server/handler/ClientRegistryHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/ClientRegistryHandler.scala @@ -20,8 +20,7 @@ private object ClientRegistryHandler { addr: String, scope: String, successRate: Double, - unavailable: Int - ) + unavailable: Int) val profileOrdering: Ordering[ClientProfile] = Ordering.by(_.successRate) @@ -72,8 +71,8 @@ private object ClientRegistryHandler { class ClientRegistryHandler( uriPrefix: String, source: MetricSource = new MetricSource, - stackRegistry: StackRegistry = ClientRegistry -) extends Service[Request, Response] { + stackRegistry: StackRegistry = ClientRegistry) + extends Service[Request, Response] { import ClientRegistryHandler._ // Search the metrics source for the stat scope that includes `clientName`. diff --git a/server/src/main/scala/com/twitter/server/handler/HistogramQueryHandler.scala b/server/src/main/scala/com/twitter/server/handler/HistogramQueryHandler.scala index 8c5a01ba..67620d95 100644 --- a/server/src/main/scala/com/twitter/server/handler/HistogramQueryHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/HistogramQueryHandler.scala @@ -21,8 +21,7 @@ object HistogramQueryHandler { average: Option[Long], min: Option[Long], max: Option[Long], - percentiles: Map[String, Long] - ) + percentiles: Map[String, Long]) /** the name and percentile thresholds used for summaries */ private val SummaryThresholds = Seq( diff --git a/server/src/main/scala/com/twitter/server/handler/ProfileResourceHandler.scala b/server/src/main/scala/com/twitter/server/handler/ProfileResourceHandler.scala index 9c0bc9d7..2982ac0b 100644 --- a/server/src/main/scala/com/twitter/server/handler/ProfileResourceHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/ProfileResourceHandler.scala @@ -10,9 +10,7 @@ import com.twitter.util.{Duration, Future, Return, Throw} import java.io.ByteArrayOutputStream import com.twitter.util.logging.Logger -class ProfileResourceHandler( - which: Thread.State -) extends Service[Request, Response] { +class ProfileResourceHandler(which: Thread.State) extends Service[Request, Response] { private[this] val log = Logger[ProfileResourceHandler] case class Params(pause: Duration, frequency: Int) diff --git a/server/src/main/scala/com/twitter/server/handler/RegistryHandler.scala b/server/src/main/scala/com/twitter/server/handler/RegistryHandler.scala index 1fa577e6..b1eacba7 100644 --- a/server/src/main/scala/com/twitter/server/handler/RegistryHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/RegistryHandler.scala @@ -42,9 +42,7 @@ class RegistryHandler extends Service[Request, Response] { newOk(jsonResponse(filterParam)) } - private[this] def filterRegistry( - filter: Option[String] - ): Registry = { + private[this] def filterRegistry(filter: Option[String]): Registry = { val registry = GlobalRegistry.get filter match { case None => registry diff --git a/server/src/main/scala/com/twitter/server/handler/ResourceHandler.scala b/server/src/main/scala/com/twitter/server/handler/ResourceHandler.scala index 916f530e..981c80c9 100644 --- a/server/src/main/scala/com/twitter/server/handler/ResourceHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/ResourceHandler.scala @@ -18,14 +18,12 @@ import scala.io.Source */ class ResourceHandler( baseRequestPath: String, - resourceResolver: PartialFunction[String, InputStream] -) extends Service[Request, Response] { + resourceResolver: PartialFunction[String, InputStream]) + extends Service[Request, Response] { // for java + backwards compatibility - def this( - basePath: String, - servingDir: String = "www" - ) = this(basePath, ResourceHandler.jarResolver(servingDir)) + def this(basePath: String, servingDir: String = "www") = + this(basePath, ResourceHandler.jarResolver(servingDir)) private[this] def meta(path: String): (Charset, String) = { val exts = path.split('.') @@ -93,10 +91,8 @@ object ResourceHandler extends JavaSingleton { * @param baseResourcePath The resource path from which this handler will serve resources. * (parameter names differ from the other constructors' for backwards compatibility) */ - def fromJar( - baseRequestPath: String, - baseResourcePath: String = "www" - ): ResourceHandler = new ResourceHandler(baseRequestPath, jarResolver(baseResourcePath)) + def fromJar(baseRequestPath: String, baseResourcePath: String = "www"): ResourceHandler = + new ResourceHandler(baseRequestPath, jarResolver(baseResourcePath)) /** * Loads resources relative to the given baseResourcePath (via java's `Class#getResourceAsStream` diff --git a/server/src/main/scala/com/twitter/server/handler/ServerRegistryHandler.scala b/server/src/main/scala/com/twitter/server/handler/ServerRegistryHandler.scala index 396f7dd8..a4ea8054 100644 --- a/server/src/main/scala/com/twitter/server/handler/ServerRegistryHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/ServerRegistryHandler.scala @@ -62,8 +62,8 @@ private object ServerRegistryHandler { class ServerRegistryHandler( uriPrefix: String, source: MetricSource = new MetricSource, - registry: StackRegistry = ServerRegistry -) extends Service[Request, Response] { + registry: StackRegistry = ServerRegistry) + extends Service[Request, Response] { // Search the metrics source for the stat scope that includes `serverName`. // The search namespace includes both "$serverName/" and "srv/$serverName" // to take into account finagle's ServerStatsReceiver. Note, unnamed servers are diff --git a/server/src/main/scala/com/twitter/server/handler/SummaryHandler.scala b/server/src/main/scala/com/twitter/server/handler/SummaryHandler.scala index 7bd1e085..cd522d78 100644 --- a/server/src/main/scala/com/twitter/server/handler/SummaryHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/SummaryHandler.scala @@ -9,10 +9,7 @@ import com.twitter.util.Future private object SummaryHandler { val TextResponse = "Visit twitter-server's admin pages via browser for a richer experience." - def render( - finagleVersion: String, - procInfo: Seq[String] - ): String = + def render(finagleVersion: String, procInfo: Seq[String]): String = s"""
diff --git a/server/src/main/scala/com/twitter/server/handler/ToggleHandler.scala b/server/src/main/scala/com/twitter/server/handler/ToggleHandler.scala index cbf1c3ee..3cbaad6f 100644 --- a/server/src/main/scala/com/twitter/server/handler/ToggleHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/ToggleHandler.scala @@ -28,8 +28,7 @@ private[handler] object ToggleHandler { id: String, fraction: Double, lastValue: Option[Boolean], - description: Option[String] - ) + description: Option[String]) /** * The components that compose the "current" `Toggle`. @@ -205,10 +204,7 @@ class ToggleHandler private[handler] (registeredLibrariesFn: () => Map[String, T * * @note package protected for testing */ - private[handler] def parsePath( - path: String, - errors: ArrayBuffer[String] - ): ParsedPath = { + private[handler] def parsePath(path: String, errors: ArrayBuffer[String]): ParsedPath = { path.split("/") match { case Array("", "admin", "toggles") => ParsedPath(None, None) @@ -275,10 +271,7 @@ class ToggleHandler private[handler] (registeredLibrariesFn: () => Map[String, T * * @note package protected for testing */ - private[handler] def deleteToggle( - libraryName: String, - id: String - ): Seq[String] = { + private[handler] def deleteToggle(libraryName: String, id: String): Seq[String] = { val errors = new ArrayBuffer[String]() val toggleMap = registeredLibrariesFn()(libraryName) log.info(s"Deleted $libraryName's toggle $id") diff --git a/server/src/main/scala/com/twitter/server/lint/SchedulerBlockingRule.scala b/server/src/main/scala/com/twitter/server/lint/SchedulerBlockingRule.scala index c1fe1aee..ee0ecc3d 100644 --- a/server/src/main/scala/com/twitter/server/lint/SchedulerBlockingRule.scala +++ b/server/src/main/scala/com/twitter/server/lint/SchedulerBlockingRule.scala @@ -19,10 +19,7 @@ object SchedulerBlockingRule { apply(Scheduler, 5.seconds) /** exposed for testing */ - private[lint] def apply( - scheduler: Scheduler, - minimumBlocking: Duration - ): Rule = { + private[lint] def apply(scheduler: Scheduler, minimumBlocking: Duration): Rule = { Rule( Category.Performance, "Blocking the Scheduler", diff --git a/server/src/main/scala/com/twitter/server/util/MetricSource.scala b/server/src/main/scala/com/twitter/server/util/MetricSource.scala index d6a68ec3..6aa9b294 100644 --- a/server/src/main/scala/com/twitter/server/util/MetricSource.scala +++ b/server/src/main/scala/com/twitter/server/util/MetricSource.scala @@ -18,8 +18,7 @@ private[server] class MetricSource( registry: () => Seq[StatsRegistry] = { () => MetricSource.registry }, - refreshInterval: Duration = 1.second -) { + refreshInterval: Duration = 1.second) { private[this] var lastRefresh = Time.now - refreshInterval private[this] var underlying: Map[String, StatEntry] = Map.empty diff --git a/server/src/main/scala/com/twitter/server/view/EndpointRegistryView.scala b/server/src/main/scala/com/twitter/server/view/EndpointRegistryView.scala index b3da5572..937bea80 100644 --- a/server/src/main/scala/com/twitter/server/view/EndpointRegistryView.scala +++ b/server/src/main/scala/com/twitter/server/view/EndpointRegistryView.scala @@ -17,9 +17,7 @@ private[server] object EndpointRegistryView { * Render current weights and socket addresses for paths in each dtab * @param dtabEntries Map of Dtabs to map of paths to Addrs */ - def render( - dtabEntries: Map[Dtab, Map[String, Addr]] - ): String = + def render(dtabEntries: Map[Dtab, Map[String, Addr]]): String = s"""
diff --git a/server/src/main/scala/com/twitter/server/view/IndexView.scala b/server/src/main/scala/com/twitter/server/view/IndexView.scala index 3ab8ee1f..9230d072 100644 --- a/server/src/main/scala/com/twitter/server/view/IndexView.scala +++ b/server/src/main/scala/com/twitter/server/view/IndexView.scala @@ -29,10 +29,7 @@ object IndexView { /** Render nav and contents into an html template. */ def render(title: String, uri: String, nav: Seq[Entry], contents: Reader[Buf]): Reader[Buf] = { - def renderNav( - ls: Seq[Entry], - sb: StringBuilder = new StringBuilder - ): String = ls match { + def renderNav(ls: Seq[Entry], sb: StringBuilder = new StringBuilder): String = ls match { case Seq() => sb.toString case Link(id, href, Post) +: rest => diff --git a/server/src/main/scala/com/twitter/server/view/StackRegistryView.scala b/server/src/main/scala/com/twitter/server/view/StackRegistryView.scala index c29906ac..f0689b3f 100644 --- a/server/src/main/scala/com/twitter/server/view/StackRegistryView.scala +++ b/server/src/main/scala/com/twitter/server/view/StackRegistryView.scala @@ -12,10 +12,7 @@ private[server] object StackRegistryView { * entries stats are exported. If present, the html page will link * to the metrics graphs for this entry. */ - def render( - entry: StackRegistry.Entry, - statScope: Option[String] - ): String = { + def render(entry: StackRegistry.Entry, statScope: Option[String]): String = { def renderParams(params: Seq[(String, String)]): String = (for ((field, value) <- params) yield { From c3b139833fa49d8007926879a7a6828e82f033a3 Mon Sep 17 00:00:00 2001 From: David Rusek Date: Tue, 11 Dec 2018 15:28:48 +0000 Subject: [PATCH 037/409] Twitter-oss: Prepare OSS libraries for release 18.12.0 Problem We want to release the next versions of our Twitter OSS libraries 18.12.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D251041 --- CHANGELOG.rst | 5 +++++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 49609872..e2a09005 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +18.12.0 +------- + +No Changes + 18.11.0 ------- diff --git a/README.md b/README.md index 9df46cb3..ba72642e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index fb326ec4..b69cc325 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "18.12.0-SNAPSHOT" +val releaseVersion = "18.12.0" val jacksonVersion = "2.9.6" val jacksonLibs = Seq( From b0b1f5f53e5591fd0faf16e7a25605d1d74ce178 Mon Sep 17 00:00:00 2001 From: David Rusek Date: Wed, 12 Dec 2018 19:20:41 +0000 Subject: [PATCH 038/409] Twitter-oss: update OSS libraries post-release to 19.1.0 Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 19.1.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D252019 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ba72642e..9df46cb3 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index b69cc325..b3bafc1b 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "18.12.0" +val releaseVersion = "19.1.0-SNAPSHOT" val jacksonVersion = "2.9.6" val jacksonLibs = Seq( From 6adcdb4f7e760d4eee338a38a0a6a8b287a2a09f Mon Sep 17 00:00:00 2001 From: Kevin Oliver Date: Wed, 19 Dec 2018 22:28:22 +0000 Subject: [PATCH 039/409] Avoid deprecated com.twitter.conversions Problem In phab D249403 these conversions were deprecated. Solution Use the new versions. JIRA Issues: CSL-7356 Differential Revision: https://phabricator.twitter.biz/D254765 --- doc/src/sphinx/code/AdvancedServer.scala | 2 +- doc/src/sphinx/code/BackwardsCompatServer.scala | 2 +- .../server/handler/logback/classic/LoggingHandlerTest.scala | 2 +- .../com/twitter/server/handler/FailedLintRuleHandler.scala | 2 +- .../scala/com/twitter/server/handler/HeapResourceHandler.scala | 2 +- .../com/twitter/server/handler/ProfileResourceHandler.scala | 2 +- .../scala/com/twitter/server/lint/SchedulerBlockingRule.scala | 2 +- .../src/main/scala/com/twitter/server/util/MetricSource.scala | 2 +- .../src/test/scala/com/twitter/server/AdminHttpServerTest.scala | 2 +- server/src/test/scala/com/twitter/server/HookTest.scala | 2 +- .../src/test/scala/com/twitter/server/TwitterServerTest.scala | 2 +- .../com/twitter/server/handler/ClientRegistryHandlerTest.scala | 2 +- .../test/scala/com/twitter/server/handler/LintHandlerTest.scala | 2 +- .../scala/com/twitter/server/handler/NoLoggingHandlerTest.scala | 2 +- .../scala/com/twitter/server/handler/ShutdownHandlerTest.scala | 2 +- .../scala/com/twitter/server/handler/TunableHandlerTest.scala | 2 +- .../com/twitter/server/lint/SchedulerBlockingRuleTest.scala | 2 +- .../src/test/scala/com/twitter/server/util/HttpUtilsTest.scala | 2 +- .../test/scala/com/twitter/server/util/MetricSourceTest.scala | 2 +- .../server/handler/slf4j/log4j12/LoggingHandlerTest.scala | 2 +- 20 files changed, 20 insertions(+), 20 deletions(-) diff --git a/doc/src/sphinx/code/AdvancedServer.scala b/doc/src/sphinx/code/AdvancedServer.scala index a0061c82..54ac20ca 100644 --- a/doc/src/sphinx/code/AdvancedServer.scala +++ b/doc/src/sphinx/code/AdvancedServer.scala @@ -1,4 +1,4 @@ -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.finagle.http.{HttpMuxer, Request, Response, Status} import com.twitter.finagle.Service import com.twitter.logging.Formatter diff --git a/doc/src/sphinx/code/BackwardsCompatServer.scala b/doc/src/sphinx/code/BackwardsCompatServer.scala index d32a65c9..dc3935c5 100644 --- a/doc/src/sphinx/code/BackwardsCompatServer.scala +++ b/doc/src/sphinx/code/BackwardsCompatServer.scala @@ -1,4 +1,4 @@ -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.finagle.http.{Request, Response, Status} import com.twitter.finagle.Service import com.twitter.server.TwitterServer diff --git a/logback-classic/src/test/scala/com/twitter/server/handler/logback/classic/LoggingHandlerTest.scala b/logback-classic/src/test/scala/com/twitter/server/handler/logback/classic/LoggingHandlerTest.scala index d9e24793..798aa97a 100644 --- a/logback-classic/src/test/scala/com/twitter/server/handler/logback/classic/LoggingHandlerTest.scala +++ b/logback-classic/src/test/scala/com/twitter/server/handler/logback/classic/LoggingHandlerTest.scala @@ -1,7 +1,7 @@ package com.twitter.server.handler.logback.classic import ch.qos.logback.classic.{Level, Logger} -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.finagle.http.Request import com.twitter.util.Await import com.twitter.util.logging.{Logger => UtilLogger} diff --git a/server/src/main/scala/com/twitter/server/handler/FailedLintRuleHandler.scala b/server/src/main/scala/com/twitter/server/handler/FailedLintRuleHandler.scala index ccde02b0..4179c709 100644 --- a/server/src/main/scala/com/twitter/server/handler/FailedLintRuleHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/FailedLintRuleHandler.scala @@ -1,6 +1,6 @@ package com.twitter.server.handler -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.finagle.Service import com.twitter.finagle.http.{Request, Response} import com.twitter.io.Buf diff --git a/server/src/main/scala/com/twitter/server/handler/HeapResourceHandler.scala b/server/src/main/scala/com/twitter/server/handler/HeapResourceHandler.scala index 64d2ac97..65453ca6 100644 --- a/server/src/main/scala/com/twitter/server/handler/HeapResourceHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/HeapResourceHandler.scala @@ -1,6 +1,6 @@ package com.twitter.server.handler -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.finagle.http.{Request, Response, Status} import com.twitter.finagle.Service import com.twitter.io.Buf diff --git a/server/src/main/scala/com/twitter/server/handler/ProfileResourceHandler.scala b/server/src/main/scala/com/twitter/server/handler/ProfileResourceHandler.scala index 2982ac0b..318fbf65 100644 --- a/server/src/main/scala/com/twitter/server/handler/ProfileResourceHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/ProfileResourceHandler.scala @@ -1,6 +1,6 @@ package com.twitter.server.handler -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.finagle.http.{Request, Response, Status} import com.twitter.finagle.Service import com.twitter.io.Buf diff --git a/server/src/main/scala/com/twitter/server/lint/SchedulerBlockingRule.scala b/server/src/main/scala/com/twitter/server/lint/SchedulerBlockingRule.scala index ee0ecc3d..19d9a9d6 100644 --- a/server/src/main/scala/com/twitter/server/lint/SchedulerBlockingRule.scala +++ b/server/src/main/scala/com/twitter/server/lint/SchedulerBlockingRule.scala @@ -1,7 +1,7 @@ package com.twitter.server.lint import com.twitter.concurrent.Scheduler -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.util.Duration import com.twitter.util.lint.{Category, Issue, Rule} diff --git a/server/src/main/scala/com/twitter/server/util/MetricSource.scala b/server/src/main/scala/com/twitter/server/util/MetricSource.scala index 6aa9b294..e3c523bd 100644 --- a/server/src/main/scala/com/twitter/server/util/MetricSource.scala +++ b/server/src/main/scala/com/twitter/server/util/MetricSource.scala @@ -1,6 +1,6 @@ package com.twitter.server.util -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.finagle.stats.{StatEntry, StatsRegistry} import com.twitter.finagle.util.LoadService import com.twitter.util.{Duration, Time} diff --git a/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala b/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala index d9944ef3..1b23abbd 100644 --- a/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala +++ b/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala @@ -1,6 +1,6 @@ package com.twitter.server -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.finagle.{Http, ListeningServer} import com.twitter.finagle.http._ import com.twitter.server.util.HttpUtils._ diff --git a/server/src/test/scala/com/twitter/server/HookTest.scala b/server/src/test/scala/com/twitter/server/HookTest.scala index c03bf5ba..dd3093cb 100644 --- a/server/src/test/scala/com/twitter/server/HookTest.scala +++ b/server/src/test/scala/com/twitter/server/HookTest.scala @@ -1,6 +1,6 @@ package com.twitter.server -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.util.Duration import org.scalatest.FunSuite import scala.collection.mutable diff --git a/server/src/test/scala/com/twitter/server/TwitterServerTest.scala b/server/src/test/scala/com/twitter/server/TwitterServerTest.scala index 9695c75b..6ccaec3f 100644 --- a/server/src/test/scala/com/twitter/server/TwitterServerTest.scala +++ b/server/src/test/scala/com/twitter/server/TwitterServerTest.scala @@ -1,6 +1,6 @@ package com.twitter.server -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.finagle.Service import com.twitter.finagle.http.{Request, Response} import com.twitter.util._ diff --git a/server/src/test/scala/com/twitter/server/handler/ClientRegistryHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/ClientRegistryHandlerTest.scala index 970081fe..0031f8cb 100644 --- a/server/src/test/scala/com/twitter/server/handler/ClientRegistryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/ClientRegistryHandlerTest.scala @@ -1,6 +1,6 @@ package com.twitter.server.handler -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.finagle.client.StackClient import com.twitter.finagle.util.StackRegistry import com.twitter.finagle.{Stack, param} diff --git a/server/src/test/scala/com/twitter/server/handler/LintHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/LintHandlerTest.scala index 6a72248d..0b7f3aad 100644 --- a/server/src/test/scala/com/twitter/server/handler/LintHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/LintHandlerTest.scala @@ -1,6 +1,6 @@ package com.twitter.server.handler -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.finagle.http.Request import com.twitter.util.Await import com.twitter.util.lint.{Category, Issue, Rule} diff --git a/server/src/test/scala/com/twitter/server/handler/NoLoggingHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/NoLoggingHandlerTest.scala index f24ba4d9..c3ecd8a6 100644 --- a/server/src/test/scala/com/twitter/server/handler/NoLoggingHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/NoLoggingHandlerTest.scala @@ -1,6 +1,6 @@ package com.twitter.server.handler -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.finagle.http.{Request, Status} import com.twitter.util.Await import org.scalatest.FunSuite diff --git a/server/src/test/scala/com/twitter/server/handler/ShutdownHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/ShutdownHandlerTest.scala index 42cd1dee..285c9649 100644 --- a/server/src/test/scala/com/twitter/server/handler/ShutdownHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/ShutdownHandlerTest.scala @@ -1,6 +1,6 @@ package com.twitter.server.handler -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.finagle.http.{Status, Request, Method} import com.twitter.server.TwitterServer import com.twitter.util.{Closable, Time, Await, Future} diff --git a/server/src/test/scala/com/twitter/server/handler/TunableHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/TunableHandlerTest.scala index e5b6bde3..fd7a7346 100644 --- a/server/src/test/scala/com/twitter/server/handler/TunableHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/TunableHandlerTest.scala @@ -1,6 +1,6 @@ package com.twitter.server.handler -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.finagle.http.{MediaType, Method, Request, Status} import com.twitter.util.Await import com.twitter.util.tunable.TunableMap diff --git a/server/src/test/scala/com/twitter/server/lint/SchedulerBlockingRuleTest.scala b/server/src/test/scala/com/twitter/server/lint/SchedulerBlockingRuleTest.scala index 93fee47b..300cfd7f 100644 --- a/server/src/test/scala/com/twitter/server/lint/SchedulerBlockingRuleTest.scala +++ b/server/src/test/scala/com/twitter/server/lint/SchedulerBlockingRuleTest.scala @@ -1,7 +1,7 @@ package com.twitter.server.lint import com.twitter.concurrent.Scheduler -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import org.junit.runner.RunWith import org.mockito.Mockito._ import org.scalatest.{FunSuite, Matchers} diff --git a/server/src/test/scala/com/twitter/server/util/HttpUtilsTest.scala b/server/src/test/scala/com/twitter/server/util/HttpUtilsTest.scala index 77b64376..b00333dd 100644 --- a/server/src/test/scala/com/twitter/server/util/HttpUtilsTest.scala +++ b/server/src/test/scala/com/twitter/server/util/HttpUtilsTest.scala @@ -1,6 +1,6 @@ package com.twitter.server.util -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.finagle.Service import com.twitter.finagle.http.{HttpMuxer, Request, Response, Status, Version} import com.twitter.io.Buf diff --git a/server/src/test/scala/com/twitter/server/util/MetricSourceTest.scala b/server/src/test/scala/com/twitter/server/util/MetricSourceTest.scala index 6e0b6329..5a381ca3 100644 --- a/server/src/test/scala/com/twitter/server/util/MetricSourceTest.scala +++ b/server/src/test/scala/com/twitter/server/util/MetricSourceTest.scala @@ -1,6 +1,6 @@ package com.twitter.server.util -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.finagle.stats.{StatsRegistry, StatEntry} import com.twitter.util.Time import org.junit.runner.RunWith diff --git a/slf4j-log4j12/src/test/scala/com/twitter/server/handler/slf4j/log4j12/LoggingHandlerTest.scala b/slf4j-log4j12/src/test/scala/com/twitter/server/handler/slf4j/log4j12/LoggingHandlerTest.scala index ce911cbc..8aecec34 100644 --- a/slf4j-log4j12/src/test/scala/com/twitter/server/handler/slf4j/log4j12/LoggingHandlerTest.scala +++ b/slf4j-log4j12/src/test/scala/com/twitter/server/handler/slf4j/log4j12/LoggingHandlerTest.scala @@ -1,6 +1,6 @@ package com.twitter.server.handler.slf4j.log4j12 -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.finagle.http.Request import com.twitter.util.Await import com.twitter.util.logging.{Logger => UtilLogger} From 8a164d46d68749310f68b207d8decd7d10986afa Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Thu, 20 Dec 2018 20:15:35 +0000 Subject: [PATCH 040/409] twitter-server: Ensure admin server and muxer get closed on shutdown Problem Shutdown doesn't propagate to admin server handlers, so it can be difficult to coordinate cleanup for the admin server. Solution Thread the close through properly. Result Shutting down a TwitterServer can now propagate down to the HttpMuxers that are set up through service loading. JIRA Issues: CSL-7407 Differential Revision: https://phabricator.twitter.biz/D254656 --- CHANGELOG.rst | 3 ++ .../com/twitter/server/AdminHttpServer.scala | 4 ++- .../com/twitter/server/util/HttpUtils.scala | 14 +++++--- .../com.twitter.finagle.http.HttpMuxHandler | 1 + .../twitter/server/AdminHttpServerTest.scala | 29 ++++++++++++++-- .../twitter/server/util/HttpUtilsTest.scala | 34 ++++++++++++++++--- 6 files changed, 73 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e2a09005..2f17bc23 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +* Propagate the admin server's shutdown to the handlers that are registered with the admin server. + ``PHAB_ID=D254656`` + 18.12.0 ------- diff --git a/server/src/main/scala/com/twitter/server/AdminHttpServer.scala b/server/src/main/scala/com/twitter/server/AdminHttpServer.scala index 7d69fecf..af83d1d5 100644 --- a/server/src/main/scala/com/twitter/server/AdminHttpServer.scala +++ b/server/src/main/scala/com/twitter/server/AdminHttpServer.scala @@ -18,7 +18,7 @@ import com.twitter.server.util.HttpUtils import com.twitter.server.view.{IndexView, NotFoundView} import com.twitter.util.lint.GlobalRules import com.twitter.util.registry.Library -import com.twitter.util.{ExecutorServiceFuturePool, Future, FuturePool, Monitor} +import com.twitter.util.{ExecutorServiceFuturePool, Future, FuturePool, Monitor, Time} import java.net.InetSocketAddress import java.util.concurrent.Executors import org.slf4j.LoggerFactory @@ -147,6 +147,8 @@ trait AdminHttpServer { self: App => def apply(request: Request): Future[Response] = HttpUtils.new404("no admin server initialized") } + + override def close(deadline: Time): Future[Unit] = underlying.close(deadline) } @volatile protected var adminHttpServer: ListeningServer = NullServer diff --git a/server/src/main/scala/com/twitter/server/util/HttpUtils.scala b/server/src/main/scala/com/twitter/server/util/HttpUtils.scala index fae8ad7d..a8dacdc7 100644 --- a/server/src/main/scala/com/twitter/server/util/HttpUtils.scala +++ b/server/src/main/scala/com/twitter/server/util/HttpUtils.scala @@ -3,7 +3,7 @@ package com.twitter.server.util import com.twitter.finagle.Service import com.twitter.finagle.http.{MediaType, Request, Response, Status, Version, HttpMuxer} import com.twitter.io.Buf -import com.twitter.util.Future +import com.twitter.util.{Future, Time, Closable} import org.jboss.netty.handler.codec.http.QueryStringDecoder import scala.collection.JavaConverters._ import scala.collection.{Map, Seq} @@ -18,10 +18,14 @@ private[server] object HttpUtils { * is empty the resulting services will be always answering with 404. */ def combine(services: Seq[HttpMuxer]): Service[Request, Response] = - Service.mk[Request, Response] { req => - val routes = services.flatMap(_.route(req)) - if (routes.nonEmpty) routes.maxBy(_.pattern.length).handler(req) - else Future.value(Response(req.version, Status.NotFound)) + new Service[Request, Response] { + def apply(req: Request): Future[Response] = { + val routes = services.flatMap(_.route(req)) + if (routes.nonEmpty) routes.maxBy(_.pattern.length).handler(req) + else Future.value(Response(req.version, Status.NotFound)) + } + + override def close(deadline: Time): Future[Unit] = Closable.all(services: _*).close(deadline) } /** diff --git a/server/src/test/resources/META-INF/services/com.twitter.finagle.http.HttpMuxHandler b/server/src/test/resources/META-INF/services/com.twitter.finagle.http.HttpMuxHandler index 1800f6ef..5706b55a 100644 --- a/server/src/test/resources/META-INF/services/com.twitter.finagle.http.HttpMuxHandler +++ b/server/src/test/resources/META-INF/services/com.twitter.finagle.http.HttpMuxHandler @@ -2,3 +2,4 @@ com.twitter.server.MockCommonsExporter com.twitter.server.MockHostMetricsExporter com.twitter.server.MockMetricsExporter com.twitter.server.MockOstrichExporter +com.twitter.server.MockClosableHandler diff --git a/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala b/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala index 1b23abbd..6bfbefd3 100644 --- a/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala +++ b/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala @@ -37,9 +37,27 @@ class MockHostMetricsExporter extends HttpMuxHandler { newOk("per host metrics!") } +object MockClosableHandler { + @volatile var closed = false +} + +class MockClosableHandler extends HttpMuxHandler { + MockClosableHandler.closed = false + + val pattern = "/admin/closable.json" + def route: Route = Route(pattern, this) + def apply(req: Request): Future[Response] = + newOk("closable!") + + override def close(deadline: Time): Future[Unit] = { + MockClosableHandler.closed = true + Future.Done + } +} + class AdminHttpServerTest extends FunSuite with Eventually with IntegrationPatience { - def checkServer(server: ListeningServer): Unit = { + def checkServer(server: ListeningServer, shadow: Boolean = false): Unit = { val port = server.boundAddress.asInstanceOf[InetSocketAddress].getPort val client = Http.client.newService(s"localhost:$port") @@ -54,6 +72,11 @@ class AdminHttpServerTest extends FunSuite with Eventually with IntegrationPatie val resp1 = Await.result(client(Request("/admin/per_host_metrics.json")), 1.second) assert(resp1.contentString.contains("per host metrics!")) + + if (!shadow) { + val resp2 = Await.result(client(Request("/admin/closable.json")), 1.second) + assert(resp2.contentString.contains("closable!")) + } } def closeServer(twitterServer: TwitterServer, adminServer: ListeningServer): Unit = { @@ -71,17 +94,19 @@ class AdminHttpServerTest extends FunSuite with Eventually with IntegrationPatie val server = new TestTwitterServer { override def main(): Unit = { checkServer(adminHttpServer) + closeServer(this, adminHttpServer) Await.result(close(5.seconds)) } } server.main(args = Array.empty[String]) + assert(MockClosableHandler.closed) } test("shadow server serves and is closed properly") { val server = new TestTwitterServer with ShadowAdminServer { override def main(): Unit = { - checkServer(shadowHttpServer) + checkServer(shadowHttpServer, shadow = true) // ShadowAdminServer does not listen for /quitquitquit // so send it to the admin server closeServer(this, adminHttpServer) diff --git a/server/src/test/scala/com/twitter/server/util/HttpUtilsTest.scala b/server/src/test/scala/com/twitter/server/util/HttpUtilsTest.scala index b00333dd..3701aef7 100644 --- a/server/src/test/scala/com/twitter/server/util/HttpUtilsTest.scala +++ b/server/src/test/scala/com/twitter/server/util/HttpUtilsTest.scala @@ -5,12 +5,9 @@ import com.twitter.finagle.Service import com.twitter.finagle.http.{HttpMuxer, Request, Response, Status, Version} import com.twitter.io.Buf import com.twitter.server.util.HttpUtils._ -import com.twitter.util.{Await, Future} -import org.junit.runner.RunWith +import com.twitter.util.{Await, Future, Time} import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class HttpUtilsTest extends FunSuite { private[this] def await[A](a: Future[A]): A = Await.result(a, 5.seconds) @@ -115,6 +112,35 @@ class HttpUtilsTest extends FunSuite { assert(res.status == Status.NotFound) } + test("combine can close the underlying services") { + var closed1 = false + var closed2 = false + + val hello = new Service[Request, Response] { + def apply(req: Request) = newOk("hello") + override def close(deadline: Time): Future[Unit] = { + closed1 = true + Future.Done + } + } + + val world = new Service[Request, Response] { + def apply(req: Request) = newOk("world") + override def close(deadline: Time): Future[Unit] = { + closed2 = true + Future.Done + } + } + val muxer0 = new HttpMuxer().withHandler("/hello", hello) + val muxer1 = new HttpMuxer().withHandler("/world", world) + + val svc = combine(Seq(muxer0, muxer1)) + await(svc.close()) + + assert(closed1) + assert(closed2) + } + test("expects") { val req1 = Request("/") req1.headerMap From 4b9598b9ace57a5fab9f322583c5bf12a0e1bc34 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Tue, 8 Jan 2019 20:13:00 +0000 Subject: [PATCH 041/409] twitter-oss: Prepare OSS libraries for release 19.1.0 Problem We want to release the next versions of our Twitter OSS libraries 19.1.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-7429 Differential Revision: https://phabricator.twitter.biz/D258182 --- CHANGELOG.rst | 3 +++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 2f17bc23..cffb376a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +19.1.0 +------- + * Propagate the admin server's shutdown to the handlers that are registered with the admin server. ``PHAB_ID=D254656`` diff --git a/README.md b/README.md index 9df46cb3..ba72642e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index b3bafc1b..63a14881 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.1.0-SNAPSHOT" +val releaseVersion = "19.1.0" val jacksonVersion = "2.9.6" val jacksonLibs = Seq( From 97cb514eeda99a86d8c7573f1074daf664f86af4 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Thu, 10 Jan 2019 22:23:53 +0000 Subject: [PATCH 042/409] twitter-oss: update OSS libraries post-release to 19.2.0 Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. JIRA Issues: CSL-7429 Differential Revision: https://phabricator.twitter.biz/D259499 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ba72642e..9df46cb3 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 63a14881..ebc8148f 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.1.0" +val releaseVersion = "19.2.0-SNAPSHOT" val jacksonVersion = "2.9.6" val jacksonLibs = Seq( From ea6976a363271f03409a954198237adf6c7f948a Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Mon, 4 Feb 2019 20:02:02 +0000 Subject: [PATCH 043/409] finagle,finatra: Remove jparker Differential Revision: https://phabricator.twitter.biz/D269058 --- OWNERS | 1 - 1 file changed, 1 deletion(-) diff --git a/OWNERS b/OWNERS index b28cdc37..dc165413 100644 --- a/OWNERS +++ b/OWNERS @@ -1,6 +1,5 @@ banderson ccoco -jparker koliver mnakamura roanta From da5ee2ebfbaca5d189098f650a0d42c2569bff51 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Mon, 11 Feb 2019 16:25:02 +0000 Subject: [PATCH 044/409] twitter-server: Made language more inclusive Problem / Solution Remove non-inclusive language from twitter-server JIRA Issues: CSL-7049 Differential Revision: https://phabricator.twitter.biz/D271737 --- doc/src/sphinx/Features.rst | 4 ++-- .../server/lint/StackRegistryDuplicatesRule.scala | 10 +++++----- .../server/lint/StackRegistryDuplicatesRuleTest.scala | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/src/sphinx/Features.rst b/doc/src/sphinx/Features.rst index be64670f..9b774566 100644 --- a/doc/src/sphinx/Features.rst +++ b/doc/src/sphinx/Features.rst @@ -220,10 +220,10 @@ Changing metrics verbosity By default, TwitterServer doesn't export `debug metrics`_ (assuming finagle-stats is used as a metrics library). Override the `com.twitter.finagle.stats.verbose` `tunable`_ (a comma-separated list of glob expressions under toggle map `finagle`) -to whitelist debug metrics, potentially without application restart. +to allowlist debug metrics, potentially without application restart. For example, the following JSON file placed in the resource folder (restart -required) as `com/twitter/tunables/finagle/instances.json` will whitelist +required) as `com/twitter/tunables/finagle/instances.json` will allowlist Netty 4 metrics for a given JVM process. :: diff --git a/server/src/main/scala/com/twitter/server/lint/StackRegistryDuplicatesRule.scala b/server/src/main/scala/com/twitter/server/lint/StackRegistryDuplicatesRule.scala index 1578e6bf..79c326a3 100644 --- a/server/src/main/scala/com/twitter/server/lint/StackRegistryDuplicatesRule.scala +++ b/server/src/main/scala/com/twitter/server/lint/StackRegistryDuplicatesRule.scala @@ -10,8 +10,8 @@ object StackRegistryDuplicatesRule { stackReg.registryName == ClientRegistry.registryName && entry.protocolLibrary == "memcached" - private[this] def isWhitelisted(whitelist: Set[String])(entry: StackRegistry.Entry): Boolean = - whitelist.contains(entry.name) + private[this] def isAllowlisted(allowlist: Set[String])(entry: StackRegistry.Entry): Boolean = + allowlist.contains(entry.name) /** * Rule that looks for registered @@ -22,10 +22,10 @@ object StackRegistryDuplicatesRule { * sharding of Memcache servers, and so are excluded. * * @param stackReg Registry used to search for duplicates. - * @param whitelist Set of entry [[com.twitter.finagle.param.Label names]] + * @param allowlist Set of entry [[com.twitter.finagle.param.Label names]] * to exclude from duplicate checking. */ - def apply(stackReg: StackRegistry, whitelist: Set[String]): Rule = { + def apply(stackReg: StackRegistry, allowlist: Set[String]): Rule = { Rule( Category.Configuration, s"Duplicate ${stackReg.registryName} StackRegistry names", @@ -34,7 +34,7 @@ object StackRegistryDuplicatesRule { ) { stackReg.registeredDuplicates .filterNot(isMemcacheClient(stackReg)) - .filterNot(isWhitelisted(whitelist)) + .filterNot(isAllowlisted(allowlist)) .map(e => Issue(s"name=${e.name} protocolLib=${e.protocolLibrary} addr=${e.addr}")) } } diff --git a/server/src/test/scala/com/twitter/server/lint/StackRegistryDuplicatesRuleTest.scala b/server/src/test/scala/com/twitter/server/lint/StackRegistryDuplicatesRuleTest.scala index 1645b536..9dd23859 100644 --- a/server/src/test/scala/com/twitter/server/lint/StackRegistryDuplicatesRuleTest.scala +++ b/server/src/test/scala/com/twitter/server/lint/StackRegistryDuplicatesRuleTest.scala @@ -39,7 +39,7 @@ class StackRegistryDuplicatesRuleTest extends FunSuite { assert(rule().size == 1) } - test("Multiple same name non-whitelisted clients does create issues") { + test("Multiple same name non-allowlisted clients does create issues") { val registry = newRegistry() val params = Stack.Params.empty + Label("thrift-test-client") registry.register("localhost:1234", StackClient.newStack, params) @@ -49,7 +49,7 @@ class StackRegistryDuplicatesRuleTest extends FunSuite { assert(rule().size == 1) } - test("Multiple same name whitelisted clients does not create issues") { + test("Multiple same name allowlisted clients does not create issues") { val registry = newRegistry() val params = Stack.Params.empty + Label("thrift-test-client") registry.register("localhost:1234", StackClient.newStack, params) From 1c654ee6b5f1fe4963c2cec9f041b046a486a2f6 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Wed, 13 Feb 2019 21:19:20 +0000 Subject: [PATCH 045/409] twitter-server: Servers Link for Listening Servers Docs Problem / Solution The `Listening Servers` section of the Admin documentation incorrectly refers to the url for seeing information about Finagle clients. Change the documentation to correctly refer to the url for seeing information about Finagle servers instead. JIRA Issues: CSL-7655 Differential Revision: https://phabricator.twitter.biz/D273494 --- doc/src/sphinx/Admin.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/sphinx/Admin.rst b/doc/src/sphinx/Admin.rst index 02a7aaff..f690fd9f 100644 --- a/doc/src/sphinx/Admin.rst +++ b/doc/src/sphinx/Admin.rst @@ -433,4 +433,4 @@ that your service is running. ~~~~~~~~~~~~~~ Surface server information exposed by Finagle. Per-server configuration parameters and -values for each module are available at `/admin/clients/`. +values for each module are available at `/admin/servers/`. From 0f5e66e60799be6949ea7e3549dce37d58e4619e Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Wed, 13 Feb 2019 23:34:31 +0000 Subject: [PATCH 046/409] twitter-server: Add Details for Watch Admin Endpoint Problem / Solution The admin documentation contains little information on the `Watch` endpoint. It can be used to view how a specific metric changes over time for a TwitterServer instance, as well as to download data for a specific metric. Let's add some additional detailed documentation for this endpoint. JIRA Issues: CSL-7243 Differential Revision: https://phabricator.twitter.biz/D273549 --- doc/src/sphinx/Admin.rst | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/doc/src/sphinx/Admin.rst b/doc/src/sphinx/Admin.rst index f690fd9f..c2dc7910 100644 --- a/doc/src/sphinx/Admin.rst +++ b/doc/src/sphinx/Admin.rst @@ -280,10 +280,45 @@ many of them represent. /admin/metrics ~~~~~~~~~~~~~~ -Watch specific metrics in real-time. +This metrics endpoint provides the ability to both watch specific metrics in +real-time, as well as download data for a specific metric. It is available in +TwitterServer regardless of the particular stats library implementation you are +using. + +Selecting the `Watch` menu item or navigating to the page directly in your web +browser displays all of the reported metrics for this TwitterServer instance. +Choosing an individual metric from the list will allow you to watch how its +value changes over time. .. image:: ../../img/metric_watch.png +If you know the exact name of the metric that you would like to watch, you can +use a fragment (named anchor) within the url to navigate directly to viewing +that specific metric. + +:: + + /admin/metrics#mediainfo/mux/failuredetector/ping + +If you would like to download data for an individual metric instead, you may use +the `m` request parameter with the full metric name. The response returned will be +JSON and include the name, delta (change since the metric was last sampled), and +value (instantaneous value) for the specific metric. + +:: + + /admin/metrics?m=mediainfo/mux/failuredetector/ping + +:: + + [ + { + "name" : "mediainfo/mux/failuredetector/ping", + "delta" : 3560.0, + "value" : 41939.0 + } + ] + /admin/metrics.json ~~~~~~~~~~~~~~~~~~~ From 8f59d7d35795a4c4a9a9a309817524a7747306f4 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Thu, 21 Feb 2019 18:48:01 +0000 Subject: [PATCH 047/409] twitter-oss: Prepare OSS libraries for release 19.2.0 Problem We want to release the next versions of our Twitter OSS libraries 19.2.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D276537 --- CHANGELOG.rst | 5 +++++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index cffb376a..537e8fde 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +19.2.0 +------- + +No Changes + 19.1.0 ------- diff --git a/README.md b/README.md index 9df46cb3..ba72642e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index ebc8148f..45f40a4b 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.2.0-SNAPSHOT" +val releaseVersion = "19.2.0" val jacksonVersion = "2.9.6" val jacksonLibs = Seq( From 30d6236a1129ee4c8a20173f74963b0ed10c1318 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Fri, 22 Feb 2019 01:02:17 +0000 Subject: [PATCH 048/409] twitter-oss: Update OSS Libraries Post-Release to 19.3.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 19.3.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. JIRA Issues: CSL-7681 Differential Revision: https://phabricator.twitter.biz/D276994 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ba72642e..9df46cb3 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 45f40a4b..0c84fe47 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.2.0" +val releaseVersion = "19.3.0-SNAPSHOT" val jacksonVersion = "2.9.6" val jacksonLibs = Seq( From 92e4dad51acd0af3ad6bf656e43504716b925eaf Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Thu, 28 Feb 2019 23:05:30 +0000 Subject: [PATCH 049/409] twitter-server: Make /histograms{,.json} endpoints consistent Problem /histograms?h=$HISTO&fmt=raw returns json as a raw array, but /histograms.json returns a big json object where the keys are the names of the histograms, and the values are the raw array. Having two different formats makes it harder to build tools that can consume either of them. Solution Change the format to match the /histograms.json style. Result /admin/histograms can now output data that looks like: { "jvm/gc/eden/pause_msec" : [ { "lowerLimit" : 8, "upperLimit" : 9, "count" : 1 }, { "lowerLimit" : 9, "upperLimit" : 10, "count" : 1 }, { "lowerLimit" : 53, "upperLimit" : 54, "count" : 1 } ] } JIRA Issues: CSL-7717 Differential Revision: https://phabricator.twitter.biz/D279779 --- CHANGELOG.rst | 4 + doc/src/sphinx/Features.rst | 126 +++++++++++++----- .../twitter-server/js/histogram-main.js | 3 +- .../handler/HistogramQueryHandler.scala | 10 +- .../twitter/server/util/JsonConverter.scala | 4 +- .../server/HistogramQueryHandlerTest.scala | 37 +++-- 6 files changed, 133 insertions(+), 51 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 537e8fde..6841423e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,10 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +* Change the /admin/histograms?h=...-style endpoints to return data in the same style as + /admin/histograms.json. This should make it easier to use tools to parse data from either + endpoint. ``PHAB_ID=D279779`` + 19.2.0 ------- diff --git a/doc/src/sphinx/Features.rst b/doc/src/sphinx/Features.rst index 9b774566..aded3e70 100644 --- a/doc/src/sphinx/Features.rst +++ b/doc/src/sphinx/Features.rst @@ -296,6 +296,52 @@ finagle-stats jar on your classpath. API Parameters ************** +There are two endpoints for histograms, `/admin/histograms.json` and `/admin/histograms`. + +`/admin/histograms.json` will always return the full list of histograms. + +:: + + $ curl http://a.twitter.server/admin/histograms.json + + { + "clnt/p2cslowservertest-server/request_latency_ms" : [ + { + "lowerLimit" : 5, + "upperLimit" : 6, + "count" : 28 + }, + { + "lowerLimit" : 6, + "upperLimit" : 7, + "count" : 9188 + }, + { + "lowerLimit" : 7, + "upperLimit" : 8, + "count" : 25164 + } + ], + "jvm/gc/eden/pause_msec" : [ + { + "lowerLimit" : 8, + "upperLimit" : 9, + "count" : 1 + }, + { + "lowerLimit" : 9, + "upperLimit" : 10, + "count" : 1 + }, + { + "lowerLimit" : 53, + "upperLimit" : 54, + "count" : 1 + } + ] + } + +`/admin/histograms` is more flexible, and can display a single histogram in many ways. The following parameters control how a histogram is displayed. 1) "h": the name of the histogram you want to see. @@ -325,19 +371,23 @@ equal to the bucket's upper limit. $ curl http://a.twitter.server/admin/histograms?h=clnt/p2cslowservertest-server/request_latency_ms&fmt=raw { - "lowerLimit" : 5, - "upperLimit" : 6, - "count" : 28 - }, - { - "lowerLimit" : 6, - "upperLimit" : 7, - "count" : 9188 - }, - { - "lowerLimit" : 7, - "upperLimit" : 8, - "count" : 25164 + "clnt/p2cslowservertest-server/request_latency_ms" : [ + { + "lowerLimit" : 5, + "upperLimit" : 6, + "count" : 28 + }, + { + "lowerLimit" : 6, + "upperLimit" : 7, + "count" : 9188 + }, + { + "lowerLimit" : 7, + "upperLimit" : 8, + "count" : 25164 + } + ] } Here's an example where we query a cumulative distribution function (CDF). Each @@ -349,29 +399,33 @@ equal to the bucket's upper limit. $ curl http://a.twitter.server/admin/histograms?h=clnt/p2cslowservertest-server/request_latency_ms&fmt=cdf { - "lowerLimit" : 5, - "upperLimit" : 6, - "percentage" : 6.444885E-5 - }, - { - "lowerLimit" : 6, - "upperLimit" : 7, - "percentage" : 0.03286891 - }, - { - "lowerLimit" : 7, - "upperLimit" : 8, - "percentage" : 0.15292539 - }, - { - "lowerLimit" : 8, - "upperLimit" : 9, - "percentage" : 0.26998794 - }, - { - "lowerLimit" : 9, - "upperLimit" : 10, - "percentage" : 0.41753477 + "clnt/p2cslowservertest-server/request_latency_ms" : [ + { + "lowerLimit" : 5, + "upperLimit" : 6, + "percentage" : 6.444885E-5 + }, + { + "lowerLimit" : 6, + "upperLimit" : 7, + "percentage" : 0.03286891 + }, + { + "lowerLimit" : 7, + "upperLimit" : 8, + "percentage" : 0.15292539 + }, + { + "lowerLimit" : 8, + "upperLimit" : 9, + "percentage" : 0.26998794 + }, + { + "lowerLimit" : 9, + "upperLimit" : 10, + "percentage" : 0.41753477 + } + ] } Plotting the "finagle/timer/deviation_ms" cumulative distribution function: diff --git a/server/src/main/resources/twitter-server/js/histogram-main.js b/server/src/main/resources/twitter-server/js/histogram-main.js index 390fcd31..2e4a2901 100644 --- a/server/src/main/resources/twitter-server/js/histogram-main.js +++ b/server/src/main/resources/twitter-server/js/histogram-main.js @@ -18,7 +18,8 @@ function histogramRefreshRequest(callback) { xhttp.setRequestHeader("Content-type", "application/x-www-form-urlencoded"); xhttp.onreadystatechange = function() { if (xhttp.readyState === 4 && xhttp.status == 200) { - refreshHistogram(JSON.parse(xhttp.responseText)); + var histoMap = JSON.parse(xhttp.responseText); + refreshHistogram(histoMap[params.h]); callback(); } else { console.log("Histogram refresh request failed with status code: " + xhttp.status + diff --git a/server/src/main/scala/com/twitter/server/handler/HistogramQueryHandler.scala b/server/src/main/scala/com/twitter/server/handler/HistogramQueryHandler.scala index 67620d95..7e87c553 100644 --- a/server/src/main/scala/com/twitter/server/handler/HistogramQueryHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/HistogramQueryHandler.scala @@ -62,10 +62,10 @@ object HistogramQueryHandler { } private[HistogramQueryHandler] def deliverData( - counts: Seq[BucketAndCount], + counts: Map[String, Seq[BucketAndCount]], transform: Seq[BucketAndCount] => Any ): String = - JsonConverter.writeToString(transform(counts)) + JsonConverter.writeToString(counts.mapValues(transform)) // Generates html for visualizing histograms private[HistogramQueryHandler] val render: String = { @@ -357,17 +357,17 @@ private[server] class HistogramQueryHandler(details: WithHistogramDetails) case Some(Seq("raw")) => jsonResponse(query, { counts: Seq[BucketAndCount] => - deliverData(counts, x => x) + deliverData(Map(query -> counts), identity) }) case Some(Seq("pdf")) => jsonResponse(query, { counts: Seq[BucketAndCount] => - deliverData(counts, x => pdf(x)) + deliverData(Map(query -> counts), x => pdf(x)) }) case Some(Seq("cdf")) => jsonResponse(query, { counts: Seq[BucketAndCount] => - deliverData(counts, x => cdf(x)) + deliverData(Map(query -> counts), x => cdf(x)) }) case _ => diff --git a/server/src/main/scala/com/twitter/server/util/JsonConverter.scala b/server/src/main/scala/com/twitter/server/util/JsonConverter.scala index 15149a9e..71dfca9b 100644 --- a/server/src/main/scala/com/twitter/server/util/JsonConverter.scala +++ b/server/src/main/scala/com/twitter/server/util/JsonConverter.scala @@ -11,9 +11,11 @@ object JsonConverter { private[this] val writer = { val factory = new MappingJsonFactory() factory.disable(JsonFactory.Feature.USE_THREAD_LOCAL_FOR_BUFFER_RECYCLING) - val mapper = new ObjectMapper(factory).registerModule(DefaultScalaModule) + val mapper = new ObjectMapper(factory) + .registerModule(DefaultScalaModule) val printer = new DefaultPrettyPrinter printer.indentArraysWith(DefaultIndenter.SYSTEM_LINEFEED_INSTANCE) + mapper.writer(printer) } diff --git a/server/src/test/scala/com/twitter/server/HistogramQueryHandlerTest.scala b/server/src/test/scala/com/twitter/server/HistogramQueryHandlerTest.scala index 136c6cb8..33278d11 100644 --- a/server/src/test/scala/com/twitter/server/HistogramQueryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/HistogramQueryHandlerTest.scala @@ -1,22 +1,28 @@ package com.twitter.server -import com.twitter.finagle.stats.InMemoryStatsReceiver +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.scala.DefaultScalaModule +import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper +import com.twitter.conversions.DurationOps._ import com.twitter.finagle.http.{Request, Response} +import com.twitter.finagle.stats.{BucketAndCount, InMemoryStatsReceiver} import com.twitter.server.handler.HistogramQueryHandler import com.twitter.util.{Await, Future} -import org.junit.runner.RunWith import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class HistogramQueryHandlerTest extends FunSuite { + def await[A](f: Future[A]): A = Await.result(f, 5.seconds) + private[this] val mapper = new ObjectMapper with ScalaObjectMapper { + registerModule(DefaultScalaModule) + } + test("histograms.json works with no stats") { val sr = new InMemoryStatsReceiver val handler = new HistogramQueryHandler(sr) val req = Request("/admin/histograms.json") val resp: Future[Response] = handler(req) - assert(Await.result(resp).contentString.contains("{ }")) + assert(await(resp).contentString.contains("{ }")) } test("histograms.json works with stats") { @@ -27,7 +33,15 @@ class HistogramQueryHandlerTest extends FunSuite { val req = Request("/admin/histograms.json") val resp: Future[Response] = handler(req) - val result = Await.result(resp).contentString + val result = await(resp).contentString + + val map = mapper.readValue[Map[String, Seq[BucketAndCount]]](result) + map("my/cool/stat") match { + case Seq(BucketAndCount(lowerLimit, upperLimit, count)) => + assert(lowerLimit == 5) + assert(upperLimit == 6) + assert(count == 1) + } assert(result.contains("my/cool/stat")) assert(result.contains(raw""""lowerLimit" : 5,""")) @@ -41,7 +55,7 @@ class HistogramQueryHandlerTest extends FunSuite { val req = Request("/admin/histograms?h=my/cool/stat&fmt=pdf") val resp: Future[Response] = handler(req) - assert(Await.result(resp).contentString.contains("not a valid histogram.")) + assert(await(resp).contentString.contains("not a valid histogram.")) } test("histograms work with a stat") { @@ -52,7 +66,14 @@ class HistogramQueryHandlerTest extends FunSuite { val req = Request("/admin/histograms?h=my/cool/stat&fmt=pdf") val resp: Future[Response] = handler(req) - val result = Await.result(resp).contentString + val result = await(resp).contentString + val map = mapper.readValue[Map[String, Seq[HistogramQueryHandler.BucketAndPercentage]]](result) + map("my/cool/stat") match { + case Seq(HistogramQueryHandler.BucketAndPercentage(lowerLimit, upperLimit, percentage)) => + assert(lowerLimit == 0) + assert(upperLimit == 1) + assert(percentage == 1.0) + } assert(result.contains(raw""""lowerLimit" : 0,""")) assert(result.contains(raw""""upperLimit" : 1,""")) assert(result.contains(raw""""percentage" : 1.0""")) From 829bd8484c9e822ccc814558badd7ac50f78146b Mon Sep 17 00:00:00 2001 From: Kevin Oliver Date: Tue, 12 Mar 2019 20:55:55 +0000 Subject: [PATCH 050/409] twitter-server: Introduce OpenCensus zPages integration Problem Users of OpenCensus would like access to it's zPages functionality. Solution Introduce a new module, `opencensus`, with a trait, `com.twitter.server.opencensus.ZPagesAdminRoutes`, that can be mixed into a `c.t.server.AdminHttpServer` to provide these pages. Result Users of this trait can visit these endpoints on their admin server and see the zPages: - /rpcz - /statz - /tracez - /traceconfigz JIRA Issues: CSL-7329 Differential Revision: https://phabricator.twitter.biz/D284757 --- build.sbt | 21 +++++ opencensus/README.md | 29 +++++++ .../scala/com/twitter/server/opencensus/BUILD | 11 +++ .../server/opencensus/ZPagesAdminRoutes.scala | 82 +++++++++++++++++++ 4 files changed, 143 insertions(+) create mode 100644 opencensus/README.md create mode 100644 opencensus/src/main/scala/com/twitter/server/opencensus/BUILD create mode 100644 opencensus/src/main/scala/com/twitter/server/opencensus/ZPagesAdminRoutes.scala diff --git a/build.sbt b/build.sbt index 0c84fe47..b3122f51 100644 --- a/build.sbt +++ b/build.sbt @@ -9,6 +9,7 @@ val jacksonLibs = Seq( "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion, "com.fasterxml.jackson.module" %% "jackson-module-scala" % jacksonVersion ) +val opencensusVersion = "0.19.1" val slf4jVersion = "1.7.21" def util(which: String) = "com.twitter" %% ("util-"+which) % releaseVersion @@ -102,6 +103,7 @@ lazy val root = (project in file(".")) .settings(noPublishSettings) .aggregate( twitterServer, + twitterServerOpenCensus, twitterServerSlf4jJdk14, twitterServerSlf4jLog4j12, twitterServerSlf4jLogbackClassic) @@ -132,6 +134,25 @@ lazy val twitterServer = (project in file("server")) ), libraryDependencies ++= jacksonLibs) +lazy val twitterServerOpenCensus = (project in file("opencensus")) + .enablePlugins( + ScalaUnidocPlugin + ) + .settings( + name := "twitter-server-opencensus", + moduleName := "twitter-server-opencensus", + sharedSettings) + .settings( + libraryDependencies ++= Seq( + finagle("core"), + finagle("http"), + "io.opencensus" % "opencensus-api" % opencensusVersion, + "io.opencensus" % "opencensus-contrib-zpages" % opencensusVersion + )) + .dependsOn( + twitterServer) + + lazy val twitterServerSlf4jJdk14 = (project in file("slf4j-jdk14")) .settings( name := "twitter-server-slf4j-jdk14", diff --git a/opencensus/README.md b/opencensus/README.md new file mode 100644 index 00000000..e72f7758 --- /dev/null +++ b/opencensus/README.md @@ -0,0 +1,29 @@ +# OpenCensus zPages + +This module integrates TwitterServer with [OpenCensus zPages](https://opencensus.io/zpages/). + +## Current State + +This library is in an experimental state. + +## Details + +By mixing in `ZPagesAdminRoutes` into a `TwitterServer`, zPages +will be served on admin routes: + + - /rpcz + - /statz + - /tracez + - /traceconfigz + +For example: + +``` +import com.twitter.server.TwitterServer +import com.twitter.server.opencensus.ZPagesAdminRoutes + +object MyServer extends TwitterServer with ZPagesAdminRoutes { + // ... +} +``` + diff --git a/opencensus/src/main/scala/com/twitter/server/opencensus/BUILD b/opencensus/src/main/scala/com/twitter/server/opencensus/BUILD new file mode 100644 index 00000000..909cd8a2 --- /dev/null +++ b/opencensus/src/main/scala/com/twitter/server/opencensus/BUILD @@ -0,0 +1,11 @@ +scala_library( + sources = globs("*.scala"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/io/opencensus:opencensus-api", + "3rdparty/jvm/io/opencensus:opencensus-contrib-zpages", + "finagle/finagle-core", + "finagle/finagle-http", + "twitter-server/server/src/main/scala", + ], +) diff --git a/opencensus/src/main/scala/com/twitter/server/opencensus/ZPagesAdminRoutes.scala b/opencensus/src/main/scala/com/twitter/server/opencensus/ZPagesAdminRoutes.scala new file mode 100644 index 00000000..9bf247b5 --- /dev/null +++ b/opencensus/src/main/scala/com/twitter/server/opencensus/ZPagesAdminRoutes.scala @@ -0,0 +1,82 @@ +package com.twitter.server.opencensus + +import com.twitter.finagle.Service +import com.twitter.finagle.http.{Request, Response, Status} +import com.twitter.io.Buf +import com.twitter.server.AdminHttpServer +import com.twitter.util.{Future, FuturePool} +import io.opencensus.contrib.zpages.{ZPageHandler, ZPageHandlers} +import java.io.ByteArrayOutputStream +import scala.collection.JavaConverters._ + +private object ZPagesAdminRoutes { + private def zpageHandlerToService( + handler: ZPageHandler, + name: String + ): Service[Request, Response] = + new Service[Request, Response] { + override def toString: String = s"ZPageHandlerService($name)" + + def apply(request: Request): Future[Response] = { + val requestParams = + request + .getParamNames().asScala.map { name => + name -> request.getParam(name) + }.toMap.asJava + + // process in a FuturePool to handle the possibility + // of zpages having blocking code. + FuturePool.unboundedPool { + val output = new ByteArrayOutputStream() + handler.emitHtml(requestParams, output) + Response(request) + .status(Status.Ok) + .content(Buf.ByteArray.Owned(output.toByteArray)) + } + } + } +} + +/** + * Mix into an [[AdminHttpServer]] to serve OpenCensus zPages on admin routes. + * + * The zPages will be available at: + * - /rpcz + * - /statz + * - /tracez + * - /traceconfigz + * + * For example: + * {{{ + * import com.twitter.server.TwitterServer + * import com.twitter.server.opencensus.ZPagesAdminRoutes + * + * object MyServer extends TwitterServer with ZPagesAdminRoutes { + * // ... + * } + * }}} + * + * @see [[https://opencensus.io/zpages/]] + */ +trait ZPagesAdminRoutes { self: AdminHttpServer => + + addAdminRoutes { + val handlers = + (ZPageHandlers.getRpczZpageHandler, "RPCz") :: + (ZPageHandlers.getStatszZPageHandler, "Statz") :: + (ZPageHandlers.getTraceConfigzZPageHandler, "Trace Configz") :: + (ZPageHandlers.getTracezZPageHandler, "Tracez") :: + Nil + + handlers.map { + case (handler, name) => + AdminHttpServer.mkRoute( + path = handler.getUrlPath, + handler = ZPagesAdminRoutes.zpageHandlerToService(handler, name), + alias = name, + group = Some("OpenCensus"), + includeInIndex = true + ) + } + } +} From 7063e7428e43725d6bfda4f7a52389c897c76117 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Wed, 13 Mar 2019 18:00:29 +0000 Subject: [PATCH 051/409] Twitter-oss: Prepare OSS libraries for release 19.3.0 Problem We want to release the next versions of our Twitter OSS libraries 19.3.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D285479 --- CHANGELOG.rst | 7 +++++-- README.md | 2 +- build.sbt | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6841423e..fbfcc816 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,17 +7,20 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +19.3.0 +------ + * Change the /admin/histograms?h=...-style endpoints to return data in the same style as /admin/histograms.json. This should make it easier to use tools to parse data from either endpoint. ``PHAB_ID=D279779`` 19.2.0 -------- +------ No Changes 19.1.0 -------- +------ * Propagate the admin server's shutdown to the handlers that are registered with the admin server. ``PHAB_ID=D254656`` diff --git a/README.md b/README.md index 9df46cb3..ba72642e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index b3122f51..2b8414c2 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.3.0-SNAPSHOT" +val releaseVersion = "19.3.0" val jacksonVersion = "2.9.6" val jacksonLibs = Seq( From 7d59c1bd46b2d96e4d0056f7860ca0344fe69247 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Wed, 13 Mar 2019 21:01:56 +0000 Subject: [PATCH 052/409] Twitter-oss: update OSS libraries post-release to 19.4.0 Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 19.4.0. - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D285562 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ba72642e..9df46cb3 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 2b8414c2..5382962b 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.3.0" +val releaseVersion = "19.4.0-SNAPSHOT" val jacksonVersion = "2.9.6" val jacksonLibs = Seq( From 1eb9b0035fabd7e84cebb438b0712ac736e4d003 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Tue, 26 Mar 2019 16:13:09 +0000 Subject: [PATCH 053/409] util, scrooge, twitter-server: Update to Scala 2.12.8 in build.sbt and Travis CI Problem/Solution Update sbt default and cross building Scala version to 2.12.8 along with the Travis CI configuration. Differential Revision: https://phabricator.twitter.biz/D291345 --- .travis.yml | 2 +- build.sbt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index e88de433..6e99dce3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,7 @@ before_cache: scala: - 2.11.12 - - 2.12.7 + - 2.12.8 jdk: - oraclejdk8 diff --git a/build.sbt b/build.sbt index 5382962b..5e99cdd6 100644 --- a/build.sbt +++ b/build.sbt @@ -26,8 +26,8 @@ lazy val noPublishSettings = Seq( lazy val sharedSettings = Seq( version := releaseVersion, organization := "com.twitter", - scalaVersion := "2.12.7", - crossScalaVersions := Seq("2.11.12", "2.12.7"), + scalaVersion := "2.12.8", + crossScalaVersions := Seq("2.11.12", "2.12.8"), libraryDependencies ++= Seq( "org.scalacheck" %% "scalacheck" % "1.13.4" % "test", "org.scalatest" %% "scalatest" % "3.0.0" % "test", From 160797c5f54f04267c4dcf752aa98cb1fa87a19f Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Tue, 26 Mar 2019 18:28:19 +0000 Subject: [PATCH 054/409] util,scrooge,fingle,twitter-server,finatra: Update sbt version from 1.1.4 to 1.2.8 Problem/Solution Update sbt version from 1.1.4 to 1.2.8 Differential Revision: https://phabricator.twitter.biz/D291366 --- project/build.properties | 2 +- sbt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/project/build.properties b/project/build.properties index 64cf32f7..c0bab049 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.1.4 +sbt.version=1.2.8 diff --git a/sbt b/sbt index 305d83a2..ad59504e 100755 --- a/sbt +++ b/sbt @@ -1,8 +1,8 @@ #!/bin/bash -sbtver=1.1.4 +sbtver=1.2.8 sbtjar=sbt-launch.jar -sbtsha128=ac6b13b709e3c0e3a8b2b3bf3031ec57660cb813 +sbtsha128=073c169c6e1a47b8ae78a7a718b907424dedab30 sbtrepo="https://repo1.maven.org/maven2/org/scala-sbt/sbt-launch" From 3772828aa734e2e4f952f73cbd01cc522db58ad8 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Tue, 2 Apr 2019 16:55:33 +0000 Subject: [PATCH 055/409] twitter-oss: Revert sbt update to 1.2.8 Problem Using `sbt 1.2.8` does not allow our projects to build with Scala 2.11.x. Even following the advice to set the root project `crossScalaVersions` to `Nil` seems to still fail (see: https://www.scala-sbt.org/1.x/docs/Cross-Build.html). Solution Revert back to `sbt 1.1.4` until we can resolve the issues with building against Scala 2.11.x or we drop support for said Scala versions. Result Green TravisCI builds. Differential Revision: https://phabricator.twitter.biz/D294569 --- project/build.properties | 2 +- sbt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/project/build.properties b/project/build.properties index c0bab049..64cf32f7 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.2.8 +sbt.version=1.1.4 diff --git a/sbt b/sbt index ad59504e..305d83a2 100755 --- a/sbt +++ b/sbt @@ -1,8 +1,8 @@ #!/bin/bash -sbtver=1.2.8 +sbtver=1.1.4 sbtjar=sbt-launch.jar -sbtsha128=073c169c6e1a47b8ae78a7a718b907424dedab30 +sbtsha128=ac6b13b709e3c0e3a8b2b3bf3031ec57660cb813 sbtrepo="https://repo1.maven.org/maven2/org/scala-sbt/sbt-launch" From e94e83006d24b26c63f279fa72ab22a95d406f4a Mon Sep 17 00:00:00 2001 From: David Rusek Date: Mon, 8 Apr 2019 17:02:13 +0000 Subject: [PATCH 056/409] twitter-server: remove ShadowAdminServer Problem/Solution/Result ShadowAdminServer has been deprecated for some time and a suitable replacement has supplanted all of it's usages. Delete it. JIRA Issues: CSL-7558 Differential Revision: https://phabricator.twitter.biz/D269149 --- CHANGELOG.rst | 5 ++ .../twitter/server/ShadowAdminServer.scala | 71 ------------------- .../twitter/server/AdminHttpServerTest.scala | 17 +---- 3 files changed, 6 insertions(+), 87 deletions(-) delete mode 100644 server/src/main/scala/com/twitter/server/ShadowAdminServer.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index fbfcc816..c328224f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Changes +~~~~~~~ + +* Remove deprecated uses of `c.t.server.ShadowAdminServer`. ``PHAB_ID=D269149`` + 19.3.0 ------ diff --git a/server/src/main/scala/com/twitter/server/ShadowAdminServer.scala b/server/src/main/scala/com/twitter/server/ShadowAdminServer.scala deleted file mode 100644 index c35c0a6c..00000000 --- a/server/src/main/scala/com/twitter/server/ShadowAdminServer.scala +++ /dev/null @@ -1,71 +0,0 @@ -package com.twitter.server - -import com.twitter.app.App -import com.twitter.concurrent.NamedPoolThreadFactory -import com.twitter.finagle.http.{HttpMuxHandler, HttpMuxer} -import com.twitter.finagle.netty4 -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.finagle.tracing.NullTracer -import com.twitter.finagle.util.LoadService -import com.twitter.finagle.{Http, ListeningServer, NullServer, param} -import com.twitter.util.logging.Logger -import java.net.InetSocketAddress -import java.util.concurrent.Executors - -/** - * An admin http server which serves requests outside the default - * finagle worker pool. This server shadows vital endpoints (ex. stats) - * that are useful for diagnostics and should be available even if the - * server becomes overwhelmed. - * - * Note, we don't serve all of /admin on a separate worker pool because - * it's important to serve certain admin endpoints in-band with the server. - * In particular, /health and /ping should be served in-band so that they - * are an accurate proxy of server health. - */ -@deprecated( - "deprecated since TwitterServer now serves everything but ping" + - " + health check outside of the global default worker pool", - since = "2017-10-04" -) -trait ShadowAdminServer { self: App with AdminHttpServer => - - @volatile protected var shadowHttpServer: ListeningServer = NullServer - val shadowAdminPort = flag( - "shadow.admin.port", - new InetSocketAddress(defaultAdminPort + 1), - "Shadow admin http server port" - ) - - premain { - val log = Logger[ShadowAdminServer] - log.info(s"Serving BlackBox http server on port ${shadowAdminPort().getPort}") - - // Ostrich, commons stats, and metrics export a `HttpMuxHandler` - val handlers = LoadService[HttpMuxHandler]() filter { handler => - handler.route.pattern == "/vars.json" || - handler.route.pattern == "/stats.json" || - handler.route.pattern == "/admin/metrics.json" || - handler.route.pattern == "/admin/per_host_metrics.json" - } - - val muxer = handlers.foldLeft(new HttpMuxer) { - case (httpMuxer, h) => httpMuxer.withHandler(h.route.pattern, h) - } - - shadowHttpServer = Http.server - .configured(param.Stats(NullStatsReceiver)) - .configured(Http.Netty4Impl) - .configured(param.Tracer(NullTracer)) - .configured( - new netty4.param.WorkerPool( - executor = Executors.newCachedThreadPool( - new NamedPoolThreadFactory("twitter-server/netty", makeDaemons = true) - ), - numWorkers = 1 - ) - ) - .serve(shadowAdminPort(), muxer) - closeOnExit(shadowHttpServer) - } -} \ No newline at end of file diff --git a/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala b/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala index 6bfbefd3..96c09886 100644 --- a/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala +++ b/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala @@ -5,7 +5,7 @@ import com.twitter.finagle.{Http, ListeningServer} import com.twitter.finagle.http._ import com.twitter.server.util.HttpUtils._ import com.twitter.util._ -import java.net.{InetAddress, InetSocketAddress} +import java.net.InetSocketAddress import org.scalatest.FunSuite import org.scalatest.concurrent.{Eventually, IntegrationPatience} @@ -103,21 +103,6 @@ class AdminHttpServerTest extends FunSuite with Eventually with IntegrationPatie assert(MockClosableHandler.closed) } - test("shadow server serves and is closed properly") { - val server = new TestTwitterServer with ShadowAdminServer { - override def main(): Unit = { - checkServer(shadowHttpServer, shadow = true) - // ShadowAdminServer does not listen for /quitquitquit - // so send it to the admin server - closeServer(this, adminHttpServer) - Await.result(close(5.seconds)) - } - } - server.shadowAdminPort.let(new InetSocketAddress(InetAddress.getLoopbackAddress, 0)) { - server.main(args = Array.empty[String]) - } - } - test("GET does not close server") { val server = new TestTwitterServer { override def main(): Unit = { From cd6589f66aba81a0ba32f1414d1d985b155ba3db Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Tue, 9 Apr 2019 04:15:57 +0000 Subject: [PATCH 057/409] finagle: Goodbye Netty 3 Problem Finagle's Netty 3 dependency and implementation no longer spark joy. Solution Drop Netty 3 dependency in Finagle and remove finagle-netty3. Thank you, Netty 3. We had a good run. JIRA Issues: CSL-7263 Differential Revision: https://phabricator.twitter.biz/D291221 --- build.sbt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 5e99cdd6..32b3ccc1 100644 --- a/build.sbt +++ b/build.sbt @@ -12,6 +12,8 @@ val jacksonLibs = Seq( val opencensusVersion = "0.19.1" val slf4jVersion = "1.7.21" +val netty3Lib = "io.netty" % "netty" % "3.10.1.Final" + def util(which: String) = "com.twitter" %% ("util-"+which) % releaseVersion def finagle(which: String) = "com.twitter" %% ("finagle-"+which) % releaseVersion @@ -130,7 +132,8 @@ lazy val twitterServer = (project in file("server")) util("registry"), util("slf4j-api"), util("slf4j-jul-bridge"), - util("tunable") + util("tunable"), + netty3Lib ), libraryDependencies ++= jacksonLibs) From 90e9ce765be1b1c30d263056af3f139d6d3be5eb Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Sat, 13 Apr 2019 00:05:11 +0000 Subject: [PATCH 058/409] twitter-server: Use HTTPS URLs Problem/Solution Update to use `https://` where possible. Differential Revision: https://phabricator.twitter.biz/D300316 --- CHANGELOG.rst | 2 +- CONFIG.ini | 4 +-- CONTRIBUTING.md | 14 ++++---- README.md | 4 +-- build.sbt | 4 +-- doc/src/sphinx/Features.rst | 34 +++++++++---------- doc/src/sphinx/_templates/sidebarintro.html | 6 ++-- doc/src/sphinx/_themes/flask/layout.html | 4 +-- .../sphinx/_themes/flask_small/layout.html | 6 ++-- doc/src/sphinx/links.txt | 2 +- .../twitter-server/css/bootstrap.min.css | 2 +- .../fonts/glyphicons-halflings-regular.svg | 6 ++-- .../twitter-server/js/bootstrap.min.js | 2 +- .../twitter/server/lint/LoggingRules.scala | 4 +-- .../com/twitter/server/view/ThreadsView.scala | 6 ++-- slf4j-log4j12/README.md | 8 ++--- 16 files changed, 53 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c328224f..925ceeb0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -610,4 +610,4 @@ Breaking API Changes 1.0.1 ----- -- Initial Release +- Initial Release \ No newline at end of file diff --git a/CONFIG.ini b/CONFIG.ini index 61d67a1f..f646bbc8 100644 --- a/CONFIG.ini +++ b/CONFIG.ini @@ -1,5 +1,3 @@ -; See http://go/CONFIG.ini - ; owned by Core System Libraries [jira] project: CSL @@ -7,4 +5,4 @@ project: CSL [docbird] project = Twitter Server project_type = library -docs_dir = ./doc/src/sphinx +docs_dir = ./doc/src/sphinx \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1c240650..d219639f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -220,17 +220,17 @@ requests that improve the existing Scaladocs! [1]: https://github.com/twitter/twitter-server/issues?direction=desc&labels=Starter&sort=created&state=open [2]: https://groups.google.com/d/forum/finaglers [3]: https://twitter.github.io/twitter-server/ -[4]: http://sphinx-doc.org/ +[4]: https://www.sphinx-doc.org/en/master/ [5]: https://github.com/sbt/sbt-site -[6]: http://sphinx-doc.org/install.html +[6]: https://www.sphinx-doc.org/en/master/usage/installation.html [7]: http://docutils.sourceforge.net/rst.html -[8]: http://docs.scala-lang.org/style/scaladoc.html +[8]: https://docs.scala-lang.org/style/scaladoc.html [9]: https://github.com/twitter/finagle/blob/master/finagle-core/src/test/java/com/twitter/finagle/AddrCompilationTest.java [es]: https://twitter.github.io/effectivescala/ -[funsuite]: http://www.scalatest.org/getting_started_with_fun_suite -[scalatest]: http://www.scalatest.org/ -[ssg]: http://docs.scala-lang.org/style/scaladoc.html +[funsuite]: https://www.scalatest.org/getting_started_with_fun_suite +[scalatest]: https://www.scalatest.org/ +[ssg]: https://docs.scala-lang.org/style/scaladoc.html [travis-ci]: https://travis-ci.org/twitter/twitter-server [changes]: https://github.com/twitter/twitter-server/blob/develop/CHANGELOG.rst [scalacheck]: https://www.scalacheck.org/ -[gendrivenprop]: http://www.scalatest.org/user_guide/generator_driven_property_checks +[gendrivenprop]: https://www.scalatest.org/user_guide/generator_driven_property_checks \ No newline at end of file diff --git a/README.md b/README.md index 9df46cb3..dfacefc9 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Browse the [user guide](https://twitter.github.io/twitter-server/). ## Releases [Releases](https://maven-badges.herokuapp.com/maven-central/com.twitter/twitter-server_2.12) -are done on an approximately monthly schedule. While [semver](http://semver.org/) +are done on an approximately monthly schedule. While [semver](https://semver.org/) is not followed, the [changelogs](CHANGELOG.rst) are detailed and include sections on public API breaks and changes in runtime behavior. @@ -44,4 +44,4 @@ The `master` branch of this repository contains the latest stable release of TwitterServer, and weekly snapshots are published to the `develop` branch. In general pull requests should be submitted against `develop`. See [CONTRIBUTING.md](https://github.com/twitter/twitter-server/blob/master/CONTRIBUTING.md) -for more details about how to contribute. +for more details about how to contribute. \ No newline at end of file diff --git a/build.sbt b/build.sbt index 32b3ccc1..5bb9927f 100644 --- a/build.sbt +++ b/build.sbt @@ -33,7 +33,7 @@ lazy val sharedSettings = Seq( libraryDependencies ++= Seq( "org.scalacheck" %% "scalacheck" % "1.13.4" % "test", "org.scalatest" %% "scalatest" % "3.0.0" % "test", - // See http://www.scala-sbt.org/0.13/docs/Testing.html#JUnit + // See https://www.scala-sbt.org/0.13/docs/Testing.html#JUnit "com.novocode" % "junit-interface" % "0.11" % "test", "org.mockito" % "mockito-all" % "1.9.5" % "test" ), @@ -74,7 +74,7 @@ lazy val sharedSettings = Seq( Apache License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 diff --git a/doc/src/sphinx/Features.rst b/doc/src/sphinx/Features.rst index aded3e70..e27d0ac4 100644 --- a/doc/src/sphinx/Features.rst +++ b/doc/src/sphinx/Features.rst @@ -57,7 +57,7 @@ Logging ------- TwitterServer uses the `Simple Logging Facade for Java (SLF4J) `__ -for framework logging. +for framework logging. .. admonition:: From the SLF4J documentation @@ -66,18 +66,18 @@ for framework logging. java.util.logging, Logback and log4j. SLF4J allows the end-user to plug in the desired logging framework at deployment time." -As such, in order to log with `TwitterServer` you must +As such, in order to log with `TwitterServer` you must `depend on an actual logging implementation `__. .. tip:: - `Logback `__ is a native `slf4j-api `__ + `Logback `__ is a native `slf4j-api `__ implementation and is the recommended logging implementation for `TwitterServer`. -Since `TwitterServer` uses the `slf4j-api `__ for logging it no -longer provides the |util-logging Logging|_ logging implementation directly. As such, logging -configuration is handled on the choosen imlpementation and by default not through flags -- except -in the case of choosing the `slf4j-jdk14` logging implementation. +Since `TwitterServer` uses the `slf4j-api `__ for logging it no +longer provides the |util-logging Logging|_ logging implementation directly. As such, logging +configuration is handled on the choosen imlpementation and by default not through flags -- except +in the case of choosing the `slf4j-jdk14` logging implementation. See the `Backwards Compatibility <#backwards-compatibility-via-slf4j-jdk14>`__ section. @@ -87,8 +87,8 @@ Backwards Compatibility (via `slf4j-jdk14`) .. includecode:: code/BackwardsCompatServer.scala :language: scala -To continue using `TwitterServer` with no changes to your logging, simply depend on the -|slf4j-jdk14|_ library which will use `slf4j-jdk14` as the |slf4j|_ logging +To continue using `TwitterServer` with no changes to your logging, simply depend on the +|twitter-server/slf4j-jdk14|_ library which will use `slf4j-jdk14` as the |slf4j|_ logging implementation, then mix in the |com.twitter.server.logging.Logging|_ to your `TwitterServer`. Example: @@ -461,7 +461,7 @@ Lifecycle Management -------------------- TwitterServer exposes endpoints to manage server lifecycle that are compatible with -`Mesos's `_ job manager: +`Mesos's `_ job manager: **/abortabortabort** Abort the process. @@ -507,14 +507,14 @@ trait is built. .. |c.t.logging.Logging| replace:: ``c.t.logging.Logging`` .. _c.t.logging.Logging: https://github.com/twitter/util/blob/70e19566e3f0a194a5364d95e04ce7fc6cf4827c/util-logging/src/main/scala/com/twitter/logging/Logging.scala#L20 -.. |slf4j-jdk14| replace:: ``slf4j-jdk14`` -.. _slf4j-jdk14: https://github.com/twitter/tree/develop/slf4j-jdk14 +.. |twitter-server/slf4j-jdk14| replace:: ``twitter-server/slf4j-jdk14`` +.. _twitter-server/slf4j-jdk14: https://github.com/twitter/twitter-server/tree/develop/slf4j-jdk14 -.. |slf4j-log4j12| replace:: ``slf4j-log4j12`` -.. _slf4j-log4j12 : https://github.com/twitter/tree/develop/slf4j-log4j12 +.. |twitter-server/slf4j-log4j12| replace:: ``twitter-server/slf4j-log4j12`` +.. _twitter-server/slf4j-log4j12 : https://github.com/twitter/twitter-server/tree/develop/slf4j-log4j12 -.. |logback-classic| replace:: ``logback-classic`` -.. _logback-classic : https://github.com/twitter/tree/develop/logback-classic +.. |twitter-server/logback-classic| replace:: ``twitter-server/logback-classic`` +.. _twitter-server/logback-classic : https://github.com/twitter/twitter-server/tree/develop/logback-classic .. |com.twitter.server.logging.Logging| replace:: ``com.twitter.server.logging.Logging`` trait -.. _com.twitter.server.logging.Logging: https://github.com/twitter/blob/develop/slf4j-jdk14/src/main/scala/com/twitter/server/logging/Logging.scala +.. _com.twitter.server.logging.Logging: https://github.com/twitter/twitter-server/blob/develop/slf4j-jdk14/src/main/scala/com/twitter/server/logging/Logging.scala \ No newline at end of file diff --git a/doc/src/sphinx/_templates/sidebarintro.html b/doc/src/sphinx/_templates/sidebarintro.html index e3774cb8..2f672326 100644 --- a/doc/src/sphinx/_templates/sidebarintro.html +++ b/doc/src/sphinx/_templates/sidebarintro.html @@ -5,6 +5,6 @@

Useful Links

+
  • TwitterServer on GitHub
    • +
  • Issue Tracker
    • + \ No newline at end of file diff --git a/doc/src/sphinx/_themes/flask/layout.html b/doc/src/sphinx/_themes/flask/layout.html index 5caa4e29..9f018809 100644 --- a/doc/src/sphinx/_themes/flask/layout.html +++ b/doc/src/sphinx/_themes/flask/layout.html @@ -17,9 +17,9 @@ {%- block footer %}
    © Copyright {{ copyright }}. - Created using Sphinx. + Created using Sphinx.
    {% if pagename == 'index' %}
    {% endif %} -{%- endblock %} +{%- endblock %} \ No newline at end of file diff --git a/doc/src/sphinx/_themes/flask_small/layout.html b/doc/src/sphinx/_themes/flask_small/layout.html index aa1716aa..b45c8a58 100644 --- a/doc/src/sphinx/_themes/flask_small/layout.html +++ b/doc/src/sphinx/_themes/flask_small/layout.html @@ -14,9 +14,9 @@ {% block relbar1 %}{% endblock %} {% block relbar2 %} {% if theme_github_fork %} - Fork me on GitHub + Fork me on GitHub {% endif %} {% endblock %} {% block sidebar1 %}{% endblock %} -{% block sidebar2 %}{% endblock %} +{% block sidebar2 %}{% endblock %} \ No newline at end of file diff --git a/doc/src/sphinx/links.txt b/doc/src/sphinx/links.txt index c5e55c8c..3a341e48 100644 --- a/doc/src/sphinx/links.txt +++ b/doc/src/sphinx/links.txt @@ -1 +1 @@ -.. _Sphinx: http://sphinx.pocoo.org/ +.. _Sphinx: https://www.sphinx-doc.org/en/master/ \ No newline at end of file diff --git a/server/src/main/resources/twitter-server/css/bootstrap.min.css b/server/src/main/resources/twitter-server/css/bootstrap.min.css index 679272d2..45212b49 100644 --- a/server/src/main/resources/twitter-server/css/bootstrap.min.css +++ b/server/src/main/resources/twitter-server/css/bootstrap.min.css @@ -1,5 +1,5 @@ /*! - * Bootstrap v3.1.1 (http://getbootstrap.com) + * Bootstrap v3.1.1 (https://getbootstrap.com) * Copyright 2011-2014 Twitter, Inc. * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) */ diff --git a/server/src/main/resources/twitter-server/fonts/glyphicons-halflings-regular.svg b/server/src/main/resources/twitter-server/fonts/glyphicons-halflings-regular.svg index e3e2dc73..5b849823 100644 --- a/server/src/main/resources/twitter-server/fonts/glyphicons-halflings-regular.svg +++ b/server/src/main/resources/twitter-server/fonts/glyphicons-halflings-regular.svg @@ -1,6 +1,6 @@ - - + + @@ -226,4 +226,4 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/server/src/main/resources/twitter-server/js/bootstrap.min.js b/server/src/main/resources/twitter-server/js/bootstrap.min.js index b04a0e82..0cc4423d 100644 --- a/server/src/main/resources/twitter-server/js/bootstrap.min.js +++ b/server/src/main/resources/twitter-server/js/bootstrap.min.js @@ -1,5 +1,5 @@ /*! - * Bootstrap v3.1.1 (http://getbootstrap.com) + * Bootstrap v3.1.1 (https://getbootstrap.com) * Copyright 2011-2014 Twitter, Inc. * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) */ diff --git a/server/src/main/scala/com/twitter/server/lint/LoggingRules.scala b/server/src/main/scala/com/twitter/server/lint/LoggingRules.scala index 9616dfdd..f997f744 100644 --- a/server/src/main/scala/com/twitter/server/lint/LoggingRules.scala +++ b/server/src/main/scala/com/twitter/server/lint/LoggingRules.scala @@ -27,7 +27,7 @@ object LoggingRules { Category.Configuration, "Multiple Slf4j Implementations", "You should only depend on a single concrete implementation of the slf4j api. " + - "See http://www.slf4j.org/codes.html#multiple_bindings" + "See https://www.slf4j.org/codes.html#multiple_bindings" ) { // same logic as org.slf4j.LoggerFactory @@ -50,4 +50,4 @@ object LoggingRules { ) { Seq(Issue("No logging handler implementation configured.")) } -} +} \ No newline at end of file diff --git a/server/src/main/scala/com/twitter/server/view/ThreadsView.scala b/server/src/main/scala/com/twitter/server/view/ThreadsView.scala index 49b21c5e..985d9dda 100644 --- a/server/src/main/scala/com/twitter/server/view/ThreadsView.scala +++ b/server/src/main/scala/com/twitter/server/view/ThreadsView.scala @@ -16,7 +16,7 @@ private object ThreadsView { private val ScriptFooter = s"""""" private[this] val StateTooltipContents = - "See Thread.State" + + "See Thread.State" + "Javadocs for details. Idle may include threads that are 'runnable' but are not doing any real " + "work, for example threads in epoll wait." @@ -38,7 +38,7 @@ private object ThreadsView {
    @@ -152,4 +152,4 @@ private[server] class ThreadsView(all: Seq[ThreadInfo], deadlockedIds: Seq[Long] ScriptFooter } -} +} \ No newline at end of file diff --git a/slf4j-log4j12/README.md b/slf4j-log4j12/README.md index 5773372d..80ef6efa 100644 --- a/slf4j-log4j12/README.md +++ b/slf4j-log4j12/README.md @@ -1,5 +1,5 @@ TwitterServer `slf4j-log4j12` -=========================== +============================ This library provides a wrapper over the [`slf4j-log4j12`](https://www.slf4j.org/manual.html#swapping) logging implementation for use in `TwitterServer` along with the appropriate [SLF4J bridges](https://www.slf4j.org/legacy.html) @@ -8,12 +8,12 @@ for other logging implementations. NOTE: ----- -Users MUST provide a compatible version of the [Log4j 1.2](http://logging.apache.org/log4j/1.2/) logging implementation. +Users MUST provide a compatible version of the [Log4j 1.2](https://logging.apache.org/log4j/1.2/) logging implementation. Admin HTTP Interface Logging Handler ------------------------------------ -Depending on this library will install a logging handler on the [HTTP admin interface](https://twitter.github.io/Admin.html#admin-interface) +Depending on this library will install a logging handler on the [HTTP admin interface](https://twitter.github.io/twitter-server/Admin.html#admin-interface) which allows users to dynamically change `org.apache.log4j.Logger` log levels. -For more information see the `TwitterServer` user guide section on [Logging](https://twitter.github.io/Features.html#logging) +For more information see the `TwitterServer` user guide section on [Logging](https://twitter.github.io/twitter-server/Features.html#logging) \ No newline at end of file From a00e3942d95fd0a0e29e14b47fa0997c8e6b3799 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Mon, 15 Apr 2019 19:26:45 +0000 Subject: [PATCH 059/409] finagle-core, twitter-server: Add `c.t.finagle.DtabFlags` Problem There is currently no defined way for users of the `c.t.finagle.Dtab` API to append to the `Dtab.base` delegation table with a supplemental value that is passed at runtime. Solution Add the `c.t.finagle.DtabFlags` trait which defines a `c.t.app.Flag`, "dtab.add" that can provide a supplemental delegation table (`Dtab`) to append to the current server `Dtab.base`. The trait exposes a `addDtabs()` function which applies the parsed Flag value and appends it to the current `Dtab.base`. Result Users are able to augment the base delegation table (`Dtab.base`) of the server with a value passed at runtime via the command line. JIRA Issues: CSL-7766 Differential Revision: https://phabricator.twitter.biz/D297596 --- CHANGELOG.rst | 8 +++++ .../main/scala/com/twitter/server/Admin.scala | 4 +-- .../com/twitter/server/FlagResolver.scala | 5 +-- .../com/twitter/server/TwitterServer.scala | 2 ++ .../twitter/server/handler/DtabHandler.scala | 36 +++++++++++++++---- 5 files changed, 44 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 925ceeb0..9d627847 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -12,6 +12,14 @@ Changes * Remove deprecated uses of `c.t.server.ShadowAdminServer`. ``PHAB_ID=D269149`` +* Mix in the `c.t.finagle.DtabFlags` to allow servers to append to the "base" `c.t.finagle.Dtab` + delegation table. Users can now call `c.t.finagle.DtabFlags#addDtabs()` when they want to append + the parsed Flag value to the `Dtab.base` delegation table. Users should note to only call this + method _after_ Flag parsing has occurred (which is after **init** and before **premain**). + + We also update the `c.t.server.handler.DtabHandler` to always return a proper JSON response of + the currently configured `c.t.finagle.Dtab.base`. ``PHAB_ID=D297596`` + 19.3.0 ------ diff --git a/server/src/main/scala/com/twitter/server/Admin.scala b/server/src/main/scala/com/twitter/server/Admin.scala index 3a2ce979..d60f44a7 100644 --- a/server/src/main/scala/com/twitter/server/Admin.scala +++ b/server/src/main/scala/com/twitter/server/Admin.scala @@ -138,7 +138,7 @@ trait Admin { self: App with AdminHttpServer with Stats => ), Route( path = "/admin/dtab", - handler = new TextBlockView().andThen(new DtabHandler), + handler = new DtabHandler, alias = "Dtab", group = Some(Grouping.ProcessInfo), includeInIndex = true @@ -195,7 +195,7 @@ trait Admin { self: App with AdminHttpServer with Stats => ), Route( path = LoadBalancersHandler.RoutePath, - handler = new LoadBalancersHandler(), + handler = new LoadBalancersHandler, alias = "Load Balancers", group = Some(Grouping.ProcessInfo), includeInIndex = true diff --git a/server/src/main/scala/com/twitter/server/FlagResolver.scala b/server/src/main/scala/com/twitter/server/FlagResolver.scala index 6d5f61e5..cc066249 100644 --- a/server/src/main/scala/com/twitter/server/FlagResolver.scala +++ b/server/src/main/scala/com/twitter/server/FlagResolver.scala @@ -4,8 +4,9 @@ import com.twitter.app.GlobalFlag import com.twitter.finagle.{Addr, Resolver, Name} import com.twitter.util.Var -// TODO: deprecate in favor of Wily dtabs. - +@deprecated( + "Users should prefer using Dtabs which are overridable by setting the `dtab.add` flag", + "2019-04-03") object resolverMap extends GlobalFlag[Map[String, String]]( Map.empty, diff --git a/server/src/main/scala/com/twitter/server/TwitterServer.scala b/server/src/main/scala/com/twitter/server/TwitterServer.scala index 1c5e11ce..8d284d63 100644 --- a/server/src/main/scala/com/twitter/server/TwitterServer.scala +++ b/server/src/main/scala/com/twitter/server/TwitterServer.scala @@ -1,6 +1,7 @@ package com.twitter.server import com.twitter.app.App +import com.twitter.finagle.DtabFlags import com.twitter.finagle.util.DefaultTimer import com.twitter.util.Timer import com.twitter.util.logging.Logging @@ -36,6 +37,7 @@ trait TwitterServer with Slf4jBridge with Logging with Linters + with DtabFlags with Hooks with AdminHttpServer with Admin diff --git a/server/src/main/scala/com/twitter/server/handler/DtabHandler.scala b/server/src/main/scala/com/twitter/server/handler/DtabHandler.scala index 5ff1c144..f62bd9be 100644 --- a/server/src/main/scala/com/twitter/server/handler/DtabHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/DtabHandler.scala @@ -1,18 +1,40 @@ package com.twitter.server.handler +import com.twitter.finagle.http.{MediaType, Request, Response} import com.twitter.finagle.{Dtab, Service} -import com.twitter.finagle.http.{Request, Response} -import com.twitter.server.util.HttpUtils.newOk +import com.twitter.io.Buf +import com.twitter.server.util.{HttpUtils, JsonConverter} import com.twitter.util.Future /** - * Dumps a simple string representation of the current Dtab. + * Dumps a simple JSON string representation of the current [[com.twitter.finagle.Dtab.base]]. * * From the Dtab docs: A Dtab--short for delegation table--comprises a sequence - * of delegation rules. Together, these describe how to bind a - * path to an Addr. + * of delegation rules. Together, these describe how to bind a path to an Addr. + * + * {{{ + * { + * "dtab": [ + * "/srv => /srv#/production", + * "/srv => /srv#/prod", + * "/s => /srv/local", + * "/$/inet => /$/nil", + * "/zk => /$/nil" + * ] + * } + * }}} + * + * @see [[com.twitter.finagle.Dtab]] */ -class DtabHandler extends Service[Request, Response] { +final class DtabHandler extends Service[Request, Response] { def apply(req: Request): Future[Response] = - newOk(Dtab.base.toString) + HttpUtils.newResponse(contentType = MediaType.Json, content = Buf.Utf8(jsonResponse)) + + private[this] def jsonResponse: String = { + JsonConverter.writeToString(Map("dtab" -> formattedDtabEntries)) + } + + private[this] def formattedDtabEntries: Seq[String] = { + for (dentry <- Dtab.base) yield { s"${dentry.prefix.show} => ${dentry.dst.show}" } + } } From 9d58a140c1b838e2af6e9eb0df17ef98839ef73e Mon Sep 17 00:00:00 2001 From: Ian Bennett Date: Wed, 17 Apr 2019 23:10:08 +0000 Subject: [PATCH 060/409] Twitter-oss: Prepare OSS libraries for release 19.4.0 Problem We want to release the next versions of our Twitter OSS libraries 19.4.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D302314 --- CHANGELOG.rst | 3 +++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9d627847..26be7de2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +19.4.0 +------ + Changes ~~~~~~~ diff --git a/README.md b/README.md index dfacefc9..8bda9c7c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 5bb9927f..299053cd 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.4.0-SNAPSHOT" +val releaseVersion = "19.4.0" val jacksonVersion = "2.9.6" val jacksonLibs = Seq( From a383fc4b51d61f337375968a64a2dcf0ef9a599c Mon Sep 17 00:00:00 2001 From: Ian Bennett Date: Fri, 19 Apr 2019 16:17:57 +0000 Subject: [PATCH 061/409] Twitter-oss: update OSS libraries post-release to 19.5.0 Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 19.5.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D302995 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8bda9c7c..dfacefc9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 299053cd..675eba8b 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.4.0" +val releaseVersion = "19.5.0-SNAPSHOT" val jacksonVersion = "2.9.6" val jacksonLibs = Seq( From e656f03fe5026f97d3dc29d871222c576d48c4ab Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Mon, 29 Apr 2019 18:22:00 +0000 Subject: [PATCH 062/409] twitter-server/slf4j-jdk14: Update tests Problem/Solution The `TestTwitterServer` defined in the `HandlersTest` shold prefer to override the `defaultAdminPort` member rather than re-define the `admin.port` flag just to set a different default. Update the test accordingly. JIRA Issues: CSL-7829 Differential Revision: https://phabricator.twitter.biz/D306919 --- .../test/scala/com/twitter/server/HandlersTest.scala | 12 ++++-------- .../scala/com/twitter/server/LogFormatTest.scala | 7 ++----- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/slf4j-jdk14/src/test/scala/com/twitter/server/HandlersTest.scala b/slf4j-jdk14/src/test/scala/com/twitter/server/HandlersTest.scala index 819c5f23..ca19980f 100644 --- a/slf4j-jdk14/src/test/scala/com/twitter/server/HandlersTest.scala +++ b/slf4j-jdk14/src/test/scala/com/twitter/server/HandlersTest.scala @@ -4,19 +4,16 @@ import com.twitter.finagle.http.{Request, Response} import com.twitter.finagle.{Http, Service} import com.twitter.util.{Await, Future} import java.io.ByteArrayOutputStream -import java.net.{InetAddress, InetSocketAddress} +import java.net.InetSocketAddress import java.util.logging.{Logger, SimpleFormatter, StreamHandler} -import org.junit.runner.RunWith import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner import scala.collection.mutable /** Test TwitterServer which overrides the admin.port to localhost ephemeral port */ class TestTwitterServer extends TwitterServer { - override val adminPort = - flag("admin.port", new InetSocketAddress(InetAddress.getLoopbackAddress, 0), "") + override val defaultAdminPort = 0 - val bootstrapSeq = mutable.MutableList.empty[Symbol] + val bootstrapSeq: mutable.MutableList[Symbol] = mutable.MutableList.empty[Symbol] def main(): Unit = { bootstrapSeq += 'Main @@ -46,11 +43,10 @@ class MockExceptionHandler extends Service[Request, Response] { } } -@RunWith(classOf[JUnitRunner]) class HandlersTest extends FunSuite { test("Exceptions thrown in handlers include stack traces") { - val twitterServer = new TestTwitterServer { + val twitterServer: TwitterServer = new TestTwitterServer { val mockExceptionHandler = new MockExceptionHandler override def main(): Unit = { diff --git a/slf4j-jdk14/src/test/scala/com/twitter/server/LogFormatTest.scala b/slf4j-jdk14/src/test/scala/com/twitter/server/LogFormatTest.scala index 32c90d8f..9361294f 100644 --- a/slf4j-jdk14/src/test/scala/com/twitter/server/LogFormatTest.scala +++ b/slf4j-jdk14/src/test/scala/com/twitter/server/LogFormatTest.scala @@ -1,13 +1,10 @@ package com.twitter.server -import com.twitter.finagle.tracing.{Trace, SpanId, TraceId} -import com.twitter.logging.{Level => TwLevel, StringHandler} +import com.twitter.finagle.tracing.{SpanId, Trace, TraceId} +import com.twitter.logging.{StringHandler, Level => TwLevel} import java.util.logging.Logger -import org.junit.runner.RunWith import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class LogFormatTest extends FunSuite { def testStringHandler(f: (Logger, StringHandler) => Unit): Unit = { val handler = new StringHandler From e8796d9db8400f003bbfb2afe63f0f846610fb09 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Wed, 8 May 2019 21:55:14 +0000 Subject: [PATCH 063/409] all: Audit gitignore Contents Problem / Solution Our open source project gitignore files have accumulated quite a bit of cruft. Let's unify them, remove no longer necessary entries, and call out project specific items. JIRA Issues: CSL-7911 Differential Revision: https://phabricator.twitter.biz/D311962 --- .gitignore | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 5c9079fc..59f053bc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,16 @@ -target/ +*.class +*.iml +*.ipr +*.log +.DS_Store +.cache +.history +.idea/ +.lib +dist/ +lib_managed/ +out/ +project/boot/ +project/plugins/project/ sbt-launch.jar +target/ From fe231c9a1034f526584ef1cbb30d1313abf78eae Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Wed, 15 May 2019 00:09:07 +0000 Subject: [PATCH 064/409] util-app|twitter-server: Track duplicated flags and add a lint rule Problem It is possible to register the same Flag name in a `c.t.app.App` multiple times. `c.t.app.App#flag` is an instance of `c.t.app.Flags` which stores Flags in a `Map[String, Flag[_]]` keyed by Flag name where the last registered name wins. The current logic prints a notice to System.err when a duplicated Flag name is registered, however, this can lead to unintended behavior for users with little visibility into what is happening. Solution Track the registration of Flags with the same name and add a lint rule to TwitterServer, `DuplicateFlagDefinitions` which will give visibility in the form of a lint rule violation when multiple Flags with the same name are registered in the app. Result Better visibility for users when multiple Flags with the same name are registered in the app. JIRA Issues: CSL-7829 Differential Revision: https://phabricator.twitter.biz/D314410 --- CHANGELOG.rst | 7 ++ .../scala/com/twitter/server/Linters.scala | 3 +- .../lint/DuplicateFlagDefinitions.scala | 30 +++++++++ .../lint/DuplicateFlagDefinitionsTest.scala | 66 +++++++++++++++++++ 4 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 server/src/main/scala/com/twitter/server/lint/DuplicateFlagDefinitions.scala create mode 100644 server/src/test/scala/com/twitter/server/lint/DuplicateFlagDefinitionsTest.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 26be7de2..a6893938 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,13 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Changes +~~~~~~~ + +* Add `DuplicateFlagDefinitions` lint rule which is violated when multiple Flags with the same + name are added to the underlying `com.twitter.app.App#flag` `com.twitter.app.Flags` instance. + ``PHAB_ID=D314410`` + 19.4.0 ------ diff --git a/server/src/main/scala/com/twitter/server/Linters.scala b/server/src/main/scala/com/twitter/server/Linters.scala index e0b5a6a7..9056d0cb 100644 --- a/server/src/main/scala/com/twitter/server/Linters.scala +++ b/server/src/main/scala/com/twitter/server/Linters.scala @@ -30,7 +30,8 @@ trait Linters { app: App => NullStatsReceiversRule(ServerRegistry), MemcacheFailFastRule(ClientRegistry), LoggingRules.MultipleSlf4jImpls, - referenceLeakLintRule.rule() + referenceLeakLintRule.rule(), + DuplicateFlagDefinitions(app) ) } diff --git a/server/src/main/scala/com/twitter/server/lint/DuplicateFlagDefinitions.scala b/server/src/main/scala/com/twitter/server/lint/DuplicateFlagDefinitions.scala new file mode 100644 index 00000000..9abdb8af --- /dev/null +++ b/server/src/main/scala/com/twitter/server/lint/DuplicateFlagDefinitions.scala @@ -0,0 +1,30 @@ +package com.twitter.server.lint + +import com.twitter.app.App +import com.twitter.util.lint.{Category, Issue, Rule} + +object DuplicateFlagDefinitions { + + def apply(app: App): Rule = { + Rule( + Category.Configuration, + "Duplicate flags registered with the same name", + """ + |Multiple flags were registered with the same Flag name. + |Registering multiple flags with the same name can lead + |to unexpected behavior and is not recommended. + """.stripMargin + ) { + issues(app.flag.registeredDuplicates) + } + } + + private def issues(dupes: Set[String]): Seq[Issue] = + if (dupes.isEmpty) { + Nil + } else { + dupes.map { dupe => + Issue(s"Duplicate flag named: $dupe") + }.toSeq + } +} diff --git a/server/src/test/scala/com/twitter/server/lint/DuplicateFlagDefinitionsTest.scala b/server/src/test/scala/com/twitter/server/lint/DuplicateFlagDefinitionsTest.scala new file mode 100644 index 00000000..32bc87b4 --- /dev/null +++ b/server/src/test/scala/com/twitter/server/lint/DuplicateFlagDefinitionsTest.scala @@ -0,0 +1,66 @@ +package com.twitter.server.lint + +import com.twitter.app.App +import org.scalatest.FunSuite + +/** does not have any duplicate flags */ +class FlagApp extends App { + flag[String]("flag1", "This is a flag") + flag[Int]("flag2", "This is a different flag.") +} + +/** duplicates "flag1" */ +class DuplicateFlagApp extends App { + flag[String]("flag1", "This is a flag") + flag[Int]("flag1", "Same flag name, different type.") +} + +/** duplicates "flag1" and "flag2" */ +class MultipleDuplicatesFlagApp extends App { + flag[String]("flag1", "This is a String flag") + flag[Int]("flag1", "Same flag name, different type.") + flag[Boolean]("flag2", "This is a boolean flag") + flag[String]("flag2", "Same flag name, different type.") +} + +/** duplicates "flag1" three times */ +class SameFlagDuplicateFlagApp extends App { + flag[String]("flag1", "This is a String flag") + flag[Int]("flag1", "Same flag name, different type.") + flag[Boolean]("flag1", "This is a boolean flag with the same name.") + flag[String]("flag2", "Same flag name, different type.") +} + +class DuplicateFlagDefinitionsTest extends FunSuite { + + test("no duplicates") { + val app = new FlagApp + val computedRule = DuplicateFlagDefinitions(app) + assert(computedRule.apply().isEmpty) + } + + test("includes the names of the duplicates") { + val app = new DuplicateFlagApp + val computedRule = DuplicateFlagDefinitions(app) + val issues = computedRule.apply() + assert(issues.length == 1) + assert(issues.exists(_.details.contains("flag1"))) + } + + test("includes the names of the multiple duplicates") { + val app = new MultipleDuplicatesFlagApp + val computedRule = DuplicateFlagDefinitions(app) + val issues = computedRule.apply() + assert(issues.length == 2) + assert(issues.exists(_.details.contains("flag1"))) + assert(issues.exists(_.details.contains("flag2"))) + } + + test("includes the name of the one flag duplicated") { + val app = new SameFlagDuplicateFlagApp + val computedRule = DuplicateFlagDefinitions(app) + val issues = computedRule.apply() + assert(issues.length == 1) + assert(issues.exists(_.details.contains("flag1"))) + } +} From 66c60483e94fb5d38e66641319ed94309c54df04 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Thu, 16 May 2019 17:10:26 +0000 Subject: [PATCH 065/409] Update to Jackson 2.9.8 Problem/Solution Update Jackson dependencies to version `2.9.8`. JIRA Issues: CSL-7986 Differential Revision: https://phabricator.twitter.biz/D304193 --- build.sbt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index 675eba8b..d0ec0a5e 100644 --- a/build.sbt +++ b/build.sbt @@ -3,11 +3,11 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch val releaseVersion = "19.5.0-SNAPSHOT" -val jacksonVersion = "2.9.6" +val jacksonVersion = "2.9.8" val jacksonLibs = Seq( "com.fasterxml.jackson.core" % "jackson-core" % jacksonVersion, "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion, - "com.fasterxml.jackson.module" %% "jackson-module-scala" % jacksonVersion + "com.fasterxml.jackson.module" %% "jackson-module-scala" % jacksonVersion exclude("com.google.guava", "guava") ) val opencensusVersion = "0.19.1" val slf4jVersion = "1.7.21" From 203ce6f5ba0c05c79cbada8b0d3a28862c2bb7a0 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Fri, 17 May 2019 18:23:01 +0000 Subject: [PATCH 066/409] csl: Release CSL libraries 19.5 It's a release day. Let's cut a May release. JIRA Issues: CSL-8130 Differential Revision: https://phabricator.twitter.biz/D315909 --- CHANGELOG.rst | 3 +++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a6893938..c12db668 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +19.5.0 +------ + Changes ~~~~~~~ diff --git a/README.md b/README.md index dfacefc9..8bda9c7c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index d0ec0a5e..7cf5c25b 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.5.0-SNAPSHOT" +val releaseVersion = "19.5.0" val jacksonVersion = "2.9.8" val jacksonLibs = Seq( From d87cca48ab2e9789bb423bf54bbc5a0a5271ca7d Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Mon, 20 May 2019 18:29:17 +0000 Subject: [PATCH 067/409] csl: Post-release CSL libraries 19.6.0-SNAPSHOT The release is done. Move to the next version. JIRA Issues: CSL-8130 Differential Revision: https://phabricator.twitter.biz/D316720 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8bda9c7c..dfacefc9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 7cf5c25b..72823715 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.5.0" +val releaseVersion = "19.6.0-SNAPSHOT" val jacksonVersion = "2.9.8" val jacksonLibs = Seq( From 1cba44ecdcb66bf513fc0d62cf7b76beee58905d Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Tue, 21 May 2019 00:30:23 +0000 Subject: [PATCH 068/409] twitter-oss: Prepare OSS libraries for point release 19.5.1 Problem We want to release a point release version of our Twitter OSS libraries 19.5.1 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libaries for their point release. Differential Revision: https://phabricator.twitter.biz/D317491 --- CHANGELOG.rst | 5 +++++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c12db668..ed3184d0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +19.5.1 +------ + +No Changes + 19.5.0 ------ diff --git a/README.md b/README.md index dfacefc9..8bda9c7c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 72823715..ad56a340 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.6.0-SNAPSHOT" +val releaseVersion = "19.5.1" val jacksonVersion = "2.9.8" val jacksonLibs = Seq( From b912f22a785b9c6c647455e90e1c3635cc2f21ca Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Tue, 21 May 2019 19:00:36 +0000 Subject: [PATCH 069/409] twitter-oss: Update OSS libraries post-release to 19.6.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 19.6.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D317825 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8bda9c7c..dfacefc9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index ad56a340..72823715 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.5.1" +val releaseVersion = "19.6.0-SNAPSHOT" val jacksonVersion = "2.9.8" val jacksonLibs = Seq( From 19493cf01972366e980458dd393d47b9f41bed16 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Mon, 3 Jun 2019 21:08:14 +0000 Subject: [PATCH 070/409] twitter-oss: Update CONTRIBUTING docs to include section on sbt script Problem/Solution We should add a section about the local `sbt` script that we expect users to use when building the project. Differential Revision: https://phabricator.twitter.biz/D322805 --- CONTRIBUTING.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d219639f..534c7c78 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,6 +13,13 @@ you can use our build tool, [dodo](https://github.com/twitter/dodo). curl -s https://raw.githubusercontent.com/twitter/dodo/develop/bin/build | bash -s -- --no-test twitter-server ``` +## Building TwitterServer + +TwitterServer is built using [sbt][sbt]. When building please use the included +[`./sbt`](https://github.com/twitter/twitter-server/blob/develop/sbt) script which +provides a thin wrapper over [sbt][sbt] and correctly sets memory and other +settings. + If you have any questions or run into any problems, please create an issue here, chat with us in [gitter](https://gitter.im/twitter/finagle), or email the Finaglers [mailing list](https://groups.google.com/forum/#!forum/finaglers). @@ -228,6 +235,7 @@ requests that improve the existing Scaladocs! [9]: https://github.com/twitter/finagle/blob/master/finagle-core/src/test/java/com/twitter/finagle/AddrCompilationTest.java [es]: https://twitter.github.io/effectivescala/ [funsuite]: https://www.scalatest.org/getting_started_with_fun_suite +[sbt]: https://www.scala-sbt.org/ [scalatest]: https://www.scalatest.org/ [ssg]: https://docs.scala-lang.org/style/scaladoc.html [travis-ci]: https://travis-ci.org/twitter/twitter-server From 85183d21b76676de4571b85c22eae0489d1816d6 Mon Sep 17 00:00:00 2001 From: Matt Dannenberg Date: Thu, 6 Jun 2019 16:32:49 +0000 Subject: [PATCH 071/409] twitter-server: metric metadata endpoint prototype Problem Users would like to know more information about the metrics emitted by a service. The most basic info they'd like is the metric type. Solution Add an endpoint (/admin/exp/metric_metadata) which will return all the metrics and their types as well as whether or not counters are latched. Additionally users may pass in an explicit set of metrics they are curious about via a query param titled m (just like the /admin/metrics endpoint). Result Users can now query their service for type information about their metrics. JIRA Issues: CSL-8125 Differential Revision: https://phabricator.twitter.biz/D313672 --- server/src/main/scala/BUILD | 93 +++++----- .../main/scala/com/twitter/server/Admin.scala | 9 +- .../server/handler/MetricQueryHandler.scala | 6 +- .../handler/MetricTypeQueryHandler.scala | 69 +++++++ .../twitter/server/util/MetricSource.scala | 30 +++- .../handler/ClientRegistryHandlerTest.scala | 12 +- .../handler/MetricTypeQueryHandlerTest.scala | 170 ++++++++++++++++++ .../server/util/MetricSourceTest.scala | 18 +- 8 files changed, 343 insertions(+), 64 deletions(-) create mode 100644 server/src/main/scala/com/twitter/server/handler/MetricTypeQueryHandler.scala create mode 100644 server/src/test/scala/com/twitter/server/handler/MetricTypeQueryHandlerTest.scala diff --git a/server/src/main/scala/BUILD b/server/src/main/scala/BUILD index a0b6d9a5..5885e7c8 100644 --- a/server/src/main/scala/BUILD +++ b/server/src/main/scala/BUILD @@ -1,47 +1,48 @@ scala_library( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, - provides = scala_artifact( - org = "com.twitter", - name = "twitter-server", - repo = artifactory, - ), - scope = "forced", # Exposes a service-loaded class. - dependencies = [ - "3rdparty/jvm/com/fasterxml/jackson/core:jackson-annotations", - "3rdparty/jvm/com/fasterxml/jackson/core:jackson-core", - "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", - "3rdparty/jvm/com/fasterxml/jackson/module:jackson-module-scala", - "3rdparty/jvm/io/netty", - "3rdparty/jvm/io/netty:netty4", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finagle/finagle-base-http/src/main/scala", - "finagle/finagle-core/src/main/scala", - "finagle/finagle-http/src/main/scala", - "finagle/finagle-netty4-http/src/main/scala", - "finagle/finagle-netty4/src/main/scala", - "finagle/finagle-toggle/src/main/scala", - "finagle/finagle-tunable/src/main/scala", - "twitter-server/server/src/main/resources", - "util/util-app/src/main/scala", - "util/util-core/src/main/scala", - "util/util-jvm/src/main/scala", - "util/util-lint/src/main/scala", - "util/util-registry/src/main/scala", - "util/util-slf4j-api/src/main/scala", - "util/util-slf4j-jul-bridge/src/main/scala", - "util/util-stats/src/main/scala", - "util/util-tunable/src/main/scala", - ], - exports = [ - "3rdparty/jvm/com/fasterxml/jackson/core:jackson-annotations", - "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finagle/finagle-base-http/src/main/scala", - "finagle/finagle-http/src/main/scala", - "util/util-app/src/main/scala", - "util/util-core/src/main/scala", - "util/util-lint/src/main/scala", - "util/util-slf4j-api/src/main/scala", - ], -) + sources = rglobs("*.scala"), + compiler_option_sets = {"fatal_warnings"}, + provides = scala_artifact( + org = "com.twitter", + name = "twitter-server", + repo = artifactory, + ), + scope = "forced", # Exposes a service-loaded class. + dependencies = [ + "3rdparty/jvm/com/fasterxml/jackson/core:jackson-annotations", + "3rdparty/jvm/com/fasterxml/jackson/core:jackson-core", + "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", + "3rdparty/jvm/com/fasterxml/jackson/module:jackson-module-scala", + "3rdparty/jvm/io/netty", + "3rdparty/jvm/io/netty:netty4", + "3rdparty/jvm/org/slf4j:slf4j-api", + "finagle/finagle-base-http/src/main/scala", + "finagle/finagle-core/src/main/scala", + "finagle/finagle-http/src/main/scala", + "finagle/finagle-netty4-http/src/main/scala", + "finagle/finagle-netty4/src/main/scala", + "finagle/finagle-stats-core/src/main/scala", + "finagle/finagle-toggle/src/main/scala", + "finagle/finagle-tunable/src/main/scala", + "twitter-server/server/src/main/resources", + "util/util-app/src/main/scala", + "util/util-core/src/main/scala", + "util/util-jvm/src/main/scala", + "util/util-lint/src/main/scala", + "util/util-registry/src/main/scala", + "util/util-slf4j-api/src/main/scala", + "util/util-slf4j-jul-bridge/src/main/scala", + "util/util-stats/src/main/scala", + "util/util-tunable/src/main/scala", + ], + exports = [ + "3rdparty/jvm/com/fasterxml/jackson/core:jackson-annotations", + "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", + "3rdparty/jvm/org/slf4j:slf4j-api", + "finagle/finagle-base-http/src/main/scala", + "finagle/finagle-http/src/main/scala", + "util/util-app/src/main/scala", + "util/util-core/src/main/scala", + "util/util-lint/src/main/scala", + "util/util-slf4j-api/src/main/scala", + ], +) \ No newline at end of file diff --git a/server/src/main/scala/com/twitter/server/Admin.scala b/server/src/main/scala/com/twitter/server/Admin.scala index d60f44a7..ad06b6f9 100644 --- a/server/src/main/scala/com/twitter/server/Admin.scala +++ b/server/src/main/scala/com/twitter/server/Admin.scala @@ -186,6 +186,13 @@ trait Admin { self: App with AdminHttpServer with Stats => group = Some(Grouping.Metrics), includeInIndex = true ), + Route( + path = "/admin/exp/metric_metadata", + handler = new MetricTypeQueryHandler, + alias = "Metric Metadata", + group = Some(Grouping.Metrics), + includeInIndex = false + ), Route( path = Path.Clients, handler = new ClientRegistryHandler(Path.Clients), @@ -287,4 +294,4 @@ trait Admin { self: App with AdminHttpServer with Stats => } else Nil } } -} +} \ No newline at end of file diff --git a/server/src/main/scala/com/twitter/server/handler/MetricQueryHandler.scala b/server/src/main/scala/com/twitter/server/handler/MetricQueryHandler.scala index 75ae13ee..6cd9585a 100644 --- a/server/src/main/scala/com/twitter/server/handler/MetricQueryHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/MetricQueryHandler.scala @@ -1,7 +1,7 @@ package com.twitter.server.handler import com.twitter.finagle.Service -import com.twitter.finagle.http.{Request, Response} +import com.twitter.finagle.http.{MediaType, Request, Response} import com.twitter.io.Buf import com.twitter.server.util.HtmlUtils.escapeHtml import com.twitter.server.util.HttpUtils.{newResponse, parse} @@ -50,9 +50,9 @@ class MetricQueryHandler(source: MetricSource = new MetricSource) case someKeys => newResponse( - contentType = "application/json;charset=UTF-8", + contentType = MediaType.JsonUtf8, content = Buf.Utf8(JsonConverter.writeToString(query(someKeys))) ) } } -} +} \ No newline at end of file diff --git a/server/src/main/scala/com/twitter/server/handler/MetricTypeQueryHandler.scala b/server/src/main/scala/com/twitter/server/handler/MetricTypeQueryHandler.scala new file mode 100644 index 00000000..6d2d5001 --- /dev/null +++ b/server/src/main/scala/com/twitter/server/handler/MetricTypeQueryHandler.scala @@ -0,0 +1,69 @@ +package com.twitter.server.handler + +import com.twitter.finagle.Service +import com.twitter.finagle.http.{MediaType, Request, Response} +import com.twitter.io.Buf +import com.twitter.server.util.HttpUtils.{newResponse, parse} +import com.twitter.server.util.{JsonConverter, MetricSource} +import com.twitter.util.Future + +/** + * A handler which accepts metrics metadata queries via http query strings and returns + * json encoded metrics with their types and an indicator of whether or not counters are latched. + * + * This is a temporary endpoint which will be replaced by a more fleshed out metadata endpoint. + * @note Histograms are not yet supported. + * + * Example Request: + * http://$HOST:$PORT/admin/exp/metric_metadata?m=srv/http/requests&m=srv/http/pending + * + * Response: + * { + * "latched" : true, + * "metrics" : [ + * { + * "name" : "srv/http/requests", + * "kind" : "counter" + * }, + * { + * "name" : "srv/http/pending", + * "kind" : "gauge" + * } + * ] + * } + */ +class MetricTypeQueryHandler(source: MetricSource = new MetricSource) + extends Service[Request, Response] { + + private[this] def query(keys: Seq[String]) = + for (k <- keys; e <- source.getType(k)) yield e + + def apply(req: Request): Future[Response] = { + val (_, params) = parse(req.uri) + + val latched = source.hasLatchedCounters + params.getOrElse("m", Nil) match { + case Nil => + newResponse( + contentType = MediaType.JsonUtf8, + content = Buf.Utf8( + JsonConverter.writeToString( + Map( + "latched" -> latched, + "metrics" -> source.typeMap + ))) + ) + + case someKeys => + newResponse( + contentType = MediaType.JsonUtf8, + content = Buf.Utf8( + JsonConverter.writeToString( + Map( + "latched" -> latched, + "metrics" -> query(someKeys) + ))) + ) + } + } +} \ No newline at end of file diff --git a/server/src/main/scala/com/twitter/server/util/MetricSource.scala b/server/src/main/scala/com/twitter/server/util/MetricSource.scala index e3c523bd..493f63af 100644 --- a/server/src/main/scala/com/twitter/server/util/MetricSource.scala +++ b/server/src/main/scala/com/twitter/server/util/MetricSource.scala @@ -3,11 +3,13 @@ package com.twitter.server.util import com.twitter.conversions.DurationOps._ import com.twitter.finagle.stats.{StatEntry, StatsRegistry} import com.twitter.finagle.util.LoadService +import com.twitter.server.util.MetricSource.MetricTypeInfo import com.twitter.util.{Duration, Time} private[server] object MetricSource { lazy val registry = LoadService[StatsRegistry]() case class Entry(name: String, delta: Double, value: Double) + case class MetricTypeInfo(name: String, kind: String) } /** @@ -32,12 +34,30 @@ private[server] class MetricSource( } } + /** Returns whether or not the MetricSource is using latched Counters + * @note this relies on the fact that there is only one StatsRegistry and that it is + * the finagle implementation. + */ + def hasLatchedCounters(): Boolean = synchronized { + val statsRegistry = registry().headOption.getOrElse { + throw new RuntimeException("No StatsRegistries available") + } + statsRegistry.latched + } + /** Returns the entry for `key` if it exists */ def get(key: String): Option[MetricSource.Entry] = synchronized { refresh() for (s <- underlying.get(key)) yield MetricSource.Entry(key, s.delta, s.value) } + /** Returns the entry for `key` with type info if the entry exists */ + def getType(key: String): Option[MetricSource.MetricTypeInfo] = synchronized { + refresh() + for (s <- underlying.get(key)) + yield MetricSource.MetricTypeInfo(key, s.metricType) + } + /** Returns true if the map contains `key` and false otherwise. */ def contains(key: String): Boolean = synchronized { refresh() @@ -49,4 +69,12 @@ private[server] class MetricSource( refresh() underlying.keySet } -} + + /** Returns a map of metric names to their types */ + def typeMap: Iterable[MetricTypeInfo] = synchronized { + refresh() + underlying.map { + case (name, statEntry) => MetricSource.MetricTypeInfo(name, statEntry.metricType) + } + } +} \ No newline at end of file diff --git a/server/src/test/scala/com/twitter/server/handler/ClientRegistryHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/ClientRegistryHandlerTest.scala index 0031f8cb..feff2783 100644 --- a/server/src/test/scala/com/twitter/server/handler/ClientRegistryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/ClientRegistryHandlerTest.scala @@ -66,11 +66,11 @@ class ClientRegistryHandlerTest extends FunSuite { assert(Await.result(handler(req)).contentString == "") underlying = Map( - "clnt/client0/loadbalancer/adds" -> Entry(10.0, 10.0), - "clnt/client0/loadbalancer/size" -> Entry(10.0, 10.0), - "clnt/client0/loadbalancer/available" -> Entry(5.0, 5.0), - "clnt/client0/requests" -> Entry(50, 50), - "clnt/client0/failures" -> Entry(1, 1) + "clnt/client0/loadbalancer/adds" -> Entry(10.0, 10.0, "counter"), + "clnt/client0/loadbalancer/size" -> Entry(10.0, 10.0, "counter"), + "clnt/client0/loadbalancer/available" -> Entry(5.0, 5.0, "counter"), + "clnt/client0/requests" -> Entry(50, 50, "counter"), + "clnt/client0/failures" -> Entry(1, 1, "counter") ) tc.advance(2.seconds) @@ -82,4 +82,4 @@ class ClientRegistryHandlerTest extends FunSuite { assert(html.contains("98.00%")) } } -} +} \ No newline at end of file diff --git a/server/src/test/scala/com/twitter/server/handler/MetricTypeQueryHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/MetricTypeQueryHandlerTest.scala new file mode 100644 index 00000000..57a55137 --- /dev/null +++ b/server/src/test/scala/com/twitter/server/handler/MetricTypeQueryHandlerTest.scala @@ -0,0 +1,170 @@ +package com.twitter.server.handler + +import com.twitter.finagle.http.Request +import com.twitter.finagle.stats.{StatEntry, StatsRegistry} +import com.twitter.server.util.MetricSource +import com.twitter.util.{Await, Duration} +import java.util.concurrent.TimeUnit +import org.scalatest.FunSuite +import scala.collection.mutable + +class MetricTypeQueryHandlerTest extends FunSuite { + + case class TrialStat(delta: Double, value: Double, metricType: String) extends StatEntry + + trait UnlatchedRegistry extends StatsRegistry { + val latched = false + val stats = mutable.Map( + "foo" -> TrialStat(3, 4, "counter"), + "bar" -> TrialStat(3, 4, "gauge"), + "baz" -> TrialStat(3, 4, "histogram")) + } + + trait LatchedRegistry extends StatsRegistry { + val latched = true + val stats = mutable.Map( + "foo" -> TrialStat(3, 4, "counter"), + "bar" -> TrialStat(3, 4, "gauge"), + "baz" -> TrialStat(3, 4, "histogram")) + } + + val latchedStatsRegistry = new LatchedRegistry() { + override def apply(): Map[String, StatEntry] = stats.toMap + } + + val unlatchedStatsRegistry = new UnlatchedRegistry() { + override def apply(): Map[String, StatEntry] = stats.toMap + } + + val latchedMetricSource = new MetricSource( + () => { + Seq(latchedStatsRegistry) + } + ) + + val unlatchedMetricSource = new MetricSource( + () => { + Seq(unlatchedStatsRegistry) + } + ) + + private[this] val latchedHandler = new MetricTypeQueryHandler(latchedMetricSource) + private[this] val unlatchedHandler = new MetricTypeQueryHandler(unlatchedMetricSource) + + val typeRequestNoArg = Request("http://$HOST:$PORT/admin/exp/metric_metadata") + + val typeRequestWithAnArg = Request("http://$HOST:$PORT/admin/exp/metric_metadata?m=bar") + + val typeRequestWithManyArgs = Request("http://$HOST:$PORT/admin/exp/metric_metadata?m=foo&m=bar") + + val responseToNoArg = + """ + | "metrics" : [ + | { + | "name" : "baz", + | "kind" : "histogram" + | }, + | { + | "name" : "foo", + | "kind" : "counter" + | }, + | { + | "name" : "bar", + | "kind" : "gauge" + | } + | ] + | } + """.stripMargin + + val responseToAnArg = + """ + | "metrics" : [ + | { + | "name" : "bar", + | "kind" : "gauge" + | } + | ] + | } + """.stripMargin + + val responseToManyArgs = + """ + | "metrics" : [ + | { + | "name" : "foo", + | "kind" : "counter" + | }, + | { + | "name" : "bar", + | "kind" : "gauge" + | } + | ] + | } + """.stripMargin + + val testNameStart = "MetricTypeQueryHandler generates reasonable json for " + + val timeout = Duration(5, TimeUnit.SECONDS) + + // NOTE: these tests assume a specific iteration order over the registries + // and HashMaps which IS NOT a guarantee. should these tests begin to fail + // due to that, we will need a more robust approach to validation. + private[this] def assertJsonResponse(actual: String, expected: String) = { + assert(stripWhitespace(actual) == stripWhitespace(expected)) + } + + private[this] def stripWhitespace(string: String): String = + string.filter { case c => !c.isWhitespace } + + def testCase(latched: Boolean, request: Request): Unit = { + if (request == typeRequestNoArg) { + testCase(latched, request, testNameStart + "full set of metrics", responseToNoArg) + } else if (request == typeRequestWithAnArg) { + testCase(latched, request, testNameStart + "a single requested metric", responseToAnArg) + } else if (request == typeRequestWithManyArgs) { + testCase(latched, request, testNameStart + "requested subset of metrics", responseToManyArgs) + } + } + + def testCase( + latched: Boolean, + request: Request, + testName: String, + responseMetrics: String + ): Unit = { + if (latched) { + val responseStart = + """ + | { + | "latched" : true, + """.stripMargin + test(testName + " when using latched counters") { + assertJsonResponse( + responseStart + responseMetrics, + Await.result(latchedHandler(request), timeout).contentString) + } + + } else { + val responseStart = + """ + | { + | "latched" : false, + """.stripMargin + test(testName + " when using unlatched counters") { + assertJsonResponse( + responseStart + responseMetrics, + Await.result(unlatchedHandler(request), timeout).contentString) + } + + } + } + + Seq( + testCase(true, typeRequestNoArg), + testCase(true, typeRequestWithManyArgs), + testCase(true, typeRequestWithAnArg), + testCase(false, typeRequestNoArg), + testCase(false, typeRequestWithManyArgs), + testCase(false, typeRequestWithAnArg) + ) +} \ No newline at end of file diff --git a/server/src/test/scala/com/twitter/server/util/MetricSourceTest.scala b/server/src/test/scala/com/twitter/server/util/MetricSourceTest.scala index 5a381ca3..71dbed7c 100644 --- a/server/src/test/scala/com/twitter/server/util/MetricSourceTest.scala +++ b/server/src/test/scala/com/twitter/server/util/MetricSourceTest.scala @@ -9,9 +9,12 @@ import org.scalatest.junit.JUnitRunner private[server] object MetricSourceTest { class Ctx { - case class Entry(delta: Double, value: Double) extends StatEntry + case class Entry(delta: Double, value: Double, metricType: String) extends StatEntry private[twitter] var underlying = Map[String, StatEntry]() - val sr = new StatsRegistry { def apply() = underlying } + val sr = new StatsRegistry { + def apply() = underlying; + override val latched: Boolean = false + } val registry = { () => Seq(sr) } @@ -28,7 +31,7 @@ class MetricSourceTest extends FunSuite { val ctx = new Ctx import ctx._ - underlying = Map("clnt/foo/requests" -> Entry(0.0, 10.0)) + underlying = Map("clnt/foo/requests" -> Entry(0.0, 10.0, "counter")) assert(source.get("clnt/foo/requests") == None) tc.advance(1.second) @@ -42,7 +45,7 @@ class MetricSourceTest extends FunSuite { val ctx = new Ctx import ctx._ - underlying = Map("clnt/foo/requests" -> Entry(0.0, 0.0)) + underlying = Map("clnt/foo/requests" -> Entry(0.0, 0.0, "counter")) assert(source.contains("clnt/foo/requests") == false) tc.advance(1.second) @@ -55,11 +58,12 @@ class MetricSourceTest extends FunSuite { val ctx = new Ctx import ctx._ - underlying = - Map("clnt/foo/requests" -> Entry(0.0, 0.0), "clnt/foo/success" -> Entry(0.0, 0.0)) + underlying = Map( + "clnt/foo/requests" -> Entry(0.0, 0.0, "counter"), + "clnt/foo/success" -> Entry(0.0, 0.0, "counter")) assert(source.keySet == Set.empty[String]) tc.advance(1.second) assert(source.keySet == Set("clnt/foo/requests", "clnt/foo/success")) } } -} +} \ No newline at end of file From 0d80cea4b7f627c4f2730a22f679af4c10608554 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Fri, 14 Jun 2019 21:38:02 +0000 Subject: [PATCH 072/409] twitter-server: Remove Netty3 from TwitterServer Problem TwitterServer contains a dependency on Netty 3 due to its use of Netty 3's `QueryStringDecoder`. Solution Change `HttpUtils` to use Netty 4's `QueryStringDecoder` and entirely remove the dependency on Netty 3 from TwitterServer. JIRA Issues: CSL-7865 Differential Revision: https://phabricator.twitter.biz/D328148 --- CHANGELOG.rst | 5 +++++ build.sbt | 5 +---- server/src/main/scala/BUILD | 1 - .../main/scala/com/twitter/server/util/HttpUtils.scala | 10 +++++----- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ed3184d0..b7a5ba6b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Changes +~~~~~~~ + +* Remove the TwitterServer dependency on Netty 3. ``PHAB_ID=D328148`` + 19.5.1 ------ diff --git a/build.sbt b/build.sbt index 72823715..9a8ac9cb 100644 --- a/build.sbt +++ b/build.sbt @@ -12,8 +12,6 @@ val jacksonLibs = Seq( val opencensusVersion = "0.19.1" val slf4jVersion = "1.7.21" -val netty3Lib = "io.netty" % "netty" % "3.10.1.Final" - def util(which: String) = "com.twitter" %% ("util-"+which) % releaseVersion def finagle(which: String) = "com.twitter" %% ("finagle-"+which) % releaseVersion @@ -132,8 +130,7 @@ lazy val twitterServer = (project in file("server")) util("registry"), util("slf4j-api"), util("slf4j-jul-bridge"), - util("tunable"), - netty3Lib + util("tunable") ), libraryDependencies ++= jacksonLibs) diff --git a/server/src/main/scala/BUILD b/server/src/main/scala/BUILD index 5885e7c8..48cb57c5 100644 --- a/server/src/main/scala/BUILD +++ b/server/src/main/scala/BUILD @@ -12,7 +12,6 @@ scala_library( "3rdparty/jvm/com/fasterxml/jackson/core:jackson-core", "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", "3rdparty/jvm/com/fasterxml/jackson/module:jackson-module-scala", - "3rdparty/jvm/io/netty", "3rdparty/jvm/io/netty:netty4", "3rdparty/jvm/org/slf4j:slf4j-api", "finagle/finagle-base-http/src/main/scala", diff --git a/server/src/main/scala/com/twitter/server/util/HttpUtils.scala b/server/src/main/scala/com/twitter/server/util/HttpUtils.scala index a8dacdc7..5c52da2b 100644 --- a/server/src/main/scala/com/twitter/server/util/HttpUtils.scala +++ b/server/src/main/scala/com/twitter/server/util/HttpUtils.scala @@ -4,7 +4,7 @@ import com.twitter.finagle.Service import com.twitter.finagle.http.{MediaType, Request, Response, Status, Version, HttpMuxer} import com.twitter.io.Buf import com.twitter.util.{Future, Time, Closable} -import org.jboss.netty.handler.codec.http.QueryStringDecoder +import io.netty.handler.codec.http.QueryStringDecoder import scala.collection.JavaConverters._ import scala.collection.{Map, Seq} @@ -42,7 +42,7 @@ private[server] object HttpUtils { */ def expectsHtml(req: Request): Boolean = { val decoder = new QueryStringDecoder(req.uri) - decoder.getPath.endsWith(".html") || accepts(req, MediaType.Html) + decoder.path().endsWith(".html") || accepts(req, MediaType.Html) } /** @@ -50,7 +50,7 @@ private[server] object HttpUtils { */ def expectsJson(req: Request): Boolean = { val decoder = new QueryStringDecoder(req.uri) - decoder.getPath.endsWith(".json") || accepts(req, MediaType.Json) + decoder.path().endsWith(".json") || accepts(req, MediaType.Json) } /** @@ -97,7 +97,7 @@ private[server] object HttpUtils { /** Parse uri into (path, params) */ def parse(uri: String): (String, Map[String, Seq[String]]) = { val qsd = new QueryStringDecoder(uri) - val params = qsd.getParameters.asScala.mapValues { _.asScala.toSeq } - (qsd.getPath, params) + val params = qsd.parameters().asScala.mapValues(_.asScala.toSeq) + (qsd.path(), params) } } From bbf3520b78a6a70da45d1163d36a18383a7be543 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Wed, 19 Jun 2019 03:48:20 +0000 Subject: [PATCH 073/409] twitter-oss: Prepare OSS Libraries for Release 19.6.0 Problem We want to release the next version of our Twitter OSS libraries 19.6.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next release. JIRA Issues: CSL-8329 Differential Revision: https://phabricator.twitter.biz/D329712 --- CHANGELOG.rst | 3 +++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b7a5ba6b..aa033a5f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +19.6.0 +------ + Changes ~~~~~~~ diff --git a/README.md b/README.md index dfacefc9..8bda9c7c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 9a8ac9cb..f9e0abd6 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.6.0-SNAPSHOT" +val releaseVersion = "19.6.0" val jacksonVersion = "2.9.8" val jacksonLibs = Seq( From 58efe65df6eec5c5be86ac972477a0cf3fa1bfc3 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Thu, 20 Jun 2019 00:12:15 +0000 Subject: [PATCH 074/409] twitter-oss: Update OSS Libraries Post-Release to 19.7.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries: 19.7.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. JIRA Issues: CSL-8329 Differential Revision: https://phabricator.twitter.biz/D330715 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8bda9c7c..dfacefc9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index f9e0abd6..70cdfd37 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.6.0" +val releaseVersion = "19.7.0-SNAPSHOT" val jacksonVersion = "2.9.8" val jacksonLibs = Seq( From 55d6d28862f7f260f3171342ad8ca363553bac40 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Fri, 21 Jun 2019 01:14:43 +0000 Subject: [PATCH 075/409] twitter-server: Remove `c.t.server.util.TwitterStats` Problem The `c.t.server.util.TwitterStats#register` function does nothing. This in turn means that the `c.t.server.Stats` trait does not need to expose an overridable `statsReceiver` function. Solution Remove the `c.t.server.util.TwitterStats` object and update the `c.t.server.Stats` trait accordingly. Result Less dead code which looks like it may be providing utility but actually does not. JIRA Issues: CSL-7676 Differential Revision: https://phabricator.twitter.biz/D330013 --- CHANGELOG.rst | 5 +++++ server/src/main/scala/com/twitter/server/Stats.scala | 9 +-------- .../scala/com/twitter/server/util/TwitterStats.scala | 9 --------- 3 files changed, 6 insertions(+), 17 deletions(-) delete mode 100644 server/src/main/scala/com/twitter/server/util/TwitterStats.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index aa033a5f..2580b818 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Changes +~~~~~~~ + +* Remove `c.t.server.util.TwitterStats` as it is dead code. ``PHAB_ID=D330013`` + 19.6.0 ------ diff --git a/server/src/main/scala/com/twitter/server/Stats.scala b/server/src/main/scala/com/twitter/server/Stats.scala index 79166f71..b6ef06b7 100644 --- a/server/src/main/scala/com/twitter/server/Stats.scala +++ b/server/src/main/scala/com/twitter/server/Stats.scala @@ -2,21 +2,14 @@ package com.twitter.server import com.twitter.app.App import com.twitter.finagle.stats.{StatsReceiver, LoadedStatsReceiver} -import com.twitter.server.util._ trait Stats { app: App => /** - * By default this returns the global [[LoadedStatsReceiver]] instance. Take note when - * overriding this method to return the *same instance* on multiple calls otherwise you - * will get surprising stating behavior. + * This returns the global [[LoadedStatsReceiver]] instance. * * @return a [[StatsReceiver]] instance. * @see [[com.twitter.finagle.stats.LoadedStatsReceiver]] */ def statsReceiver: StatsReceiver = LoadedStatsReceiver - - premain { - TwitterStats.register(statsReceiver) - } } diff --git a/server/src/main/scala/com/twitter/server/util/TwitterStats.scala b/server/src/main/scala/com/twitter/server/util/TwitterStats.scala deleted file mode 100644 index 15a40b90..00000000 --- a/server/src/main/scala/com/twitter/server/util/TwitterStats.scala +++ /dev/null @@ -1,9 +0,0 @@ -package com.twitter.server.util - -import com.twitter.finagle.stats.StatsReceiver - -object TwitterStats { - - def register(statsReceiver: StatsReceiver): Unit = () - -} From 2c233bd5cc4590c7fa91de29c02a6424ad3a70fe Mon Sep 17 00:00:00 2001 From: David Rusek Date: Wed, 26 Jun 2019 20:07:07 +0000 Subject: [PATCH 076/409] finagle/twitter-server: Add admin endpoint for info about clients connected to servers Problem There is no way to get information about clients currently connected to a finagle server. Solution/Result Add an admin endpoint to list basic information about each client connected, as well as information about the connection's encryption status, for each listening server. JIRA Issues: CSL-8104 Differential Revision: https://phabricator.twitter.biz/D329940 --- CHANGELOG.rst | 6 + doc/src/sphinx/Admin.rst | 6 + .../main/scala/com/twitter/server/Admin.scala | 7 ++ .../handler/AttachedClientsHandler.scala | 92 +++++++++++++++ .../handler/AttachedClientsHandlerTest.scala | 106 ++++++++++++++++++ 5 files changed, 217 insertions(+) create mode 100644 server/src/main/scala/com/twitter/server/handler/AttachedClientsHandler.scala create mode 100644 server/src/test/scala/com/twitter/server/handler/AttachedClientsHandlerTest.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 2580b818..838e5ce4 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -20,6 +20,12 @@ Changes * Remove the TwitterServer dependency on Netty 3. ``PHAB_ID=D328148`` +New Features +~~~~~~~~~~~~ + +* Added an admin page, /admin/servers/connections.json with details about incoming connections, + including encryption status and remote principal ``PHAB_ID=D329940`` + 19.5.1 ------ diff --git a/doc/src/sphinx/Admin.rst b/doc/src/sphinx/Admin.rst index c2dc7910..884bdc14 100644 --- a/doc/src/sphinx/Admin.rst +++ b/doc/src/sphinx/Admin.rst @@ -469,3 +469,9 @@ that your service is running. Surface server information exposed by Finagle. Per-server configuration parameters and values for each module are available at `/admin/servers/`. + +/admin/servers/connections +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Expose information about currently connected clients including encryption status, +if available. diff --git a/server/src/main/scala/com/twitter/server/Admin.scala b/server/src/main/scala/com/twitter/server/Admin.scala index ad06b6f9..735dbe6e 100644 --- a/server/src/main/scala/com/twitter/server/Admin.scala +++ b/server/src/main/scala/com/twitter/server/Admin.scala @@ -263,6 +263,13 @@ trait Admin { self: App with AdminHttpServer with Stats => alias = "Favicon", group = None, includeInIndex = false + ), + Route( + path = Path.Servers + "connections/", + handler = new AttachedClientsHandler(), + alias = "Incoming Connections", + group = Some(Grouping.Utilities), + includeInIndex = true ) ).map(Route.isolate) diff --git a/server/src/main/scala/com/twitter/server/handler/AttachedClientsHandler.scala b/server/src/main/scala/com/twitter/server/handler/AttachedClientsHandler.scala new file mode 100644 index 00000000..1df9c738 --- /dev/null +++ b/server/src/main/scala/com/twitter/server/handler/AttachedClientsHandler.scala @@ -0,0 +1,92 @@ +package com.twitter.server.handler + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include +import com.fasterxml.jackson.databind.PropertyNamingStrategy +import com.fasterxml.jackson.databind.annotation.JsonNaming +import com.twitter.finagle.Service +import com.twitter.finagle.http.{Request, Response} +import com.twitter.finagle.server.ServerRegistry +import com.twitter.server.util.JsonConverter +import com.twitter.util.Future +import java.net.SocketAddress + +private[handler] object AttachedClientsHandler { + + case class ClientConnectionEntry(address: SocketAddress, ssl: Option[ClientSslInfo]) + + @JsonNaming(classOf[PropertyNamingStrategy.SnakeCaseStrategy]) + case class PeerCertInfo(commonName: String) + + @JsonNaming(classOf[PropertyNamingStrategy.SnakeCaseStrategy]) + @JsonInclude(value = Include.NON_ABSENT, content = Include.NON_ABSENT) + case class ClientSslInfo( + sessionId: String, + cipherSuite: String, + peerCertificate: Option[PeerCertInfo]) + + case class ServerConnectionInfo(address: SocketAddress, clients: Seq[ClientConnectionEntry]) + + case class AttachedClientsConnectionInfo(servers: Seq[ServerConnectionInfo]) + + private def render(serverRegistry: ServerRegistry): AttachedClientsConnectionInfo = { + AttachedClientsConnectionInfo(serverRegistry.serverAddresses.flatMap { serverAddress => + val connectionRegistry = serverRegistry.connectionRegistry(serverAddress) + Some( + ServerConnectionInfo( + address = serverAddress, + clients = connectionRegistry.iterator.map { + clientConnection => + ClientConnectionEntry( + address = clientConnection.remoteAddress, + ssl = if (!clientConnection.sslSessionInfo.usingSsl) { + None + } else { + Some(ClientSslInfo( + sessionId = clientConnection.sslSessionInfo.sessionId, + cipherSuite = clientConnection.sslSessionInfo.cipherSuite, + peerCertificate = clientConnection.sslSessionInfo.peerCertificates.headOption + .map { peerCertificate => + PeerCertInfo(peerCertificate.getSubjectDN.getName) + } + )) + } + ) + }.toList + )) + }) + } +} + +/** + * A handler that reports information about connected clients, by server. For example: + * + * {{{ + * { + * "servers": [ + * { + * "address": "127.0.0.1:8080", + * "clients": [ + * { + * "address": "127.0.0.1:9090", + * "ssl": { + * "session_id": "sessionid", + * "cipher_suite": "cipher?sweeeeet!", + * "peer_certificate": { + * "common_name": "remoteprincipal" + * } + * } + * } + * ] + * } + * ] + * } + * }}} + */ +class AttachedClientsHandler(serverRegistry: ServerRegistry = ServerRegistry) + extends Service[Request, Response] { + def apply(request: Request): Future[Response] = { + val doc = AttachedClientsHandler.render(serverRegistry) + Future.value(JsonConverter(doc)) + } +} diff --git a/server/src/test/scala/com/twitter/server/handler/AttachedClientsHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/AttachedClientsHandlerTest.scala new file mode 100644 index 00000000..4840a810 --- /dev/null +++ b/server/src/test/scala/com/twitter/server/handler/AttachedClientsHandlerTest.scala @@ -0,0 +1,106 @@ +package com.twitter.server.handler + +import com.twitter.conversions.DurationOps._ +import com.twitter.finagle.http.{Request, Status} +import com.twitter.finagle.server.ServerRegistry +import com.twitter.finagle.service.StatsFilter +import com.twitter.finagle.ssl.session.SslSessionInfo +import com.twitter.finagle.{ClientConnection, Service, ServiceFactory, Stack, param} +import com.twitter.io.Buf +import com.twitter.util.{Await, Future, Time} +import java.net.{InetSocketAddress, SocketAddress} +import java.security.Principal +import java.security.cert.X509Certificate +import javax.net.ssl.SSLSession +import org.mockito.Mockito +import org.scalatest.FunSuite + +object AttachedClientsHandlerTest { + def await[A](f: Future[A]): A = Await.result(f, 2.seconds) + + val remoteSocketAddress = InetSocketAddress.createUnresolved("/127.0.0.1", 9090) + + def registerServer(registry: ServerRegistry, addr: SocketAddress, name: String): Unit = { + val ok = ServiceFactory.const(Service.const(Future.value("ok"))) + val leaf = StatsFilter.module.toStack(Stack.leaf(Stack.Role("ok"), ok)) + + registry.register(addr.toString, leaf, Stack.Params.empty + param.Label(name)) + + val peerCertificate = Mockito.mock(classOf[X509Certificate]) + val remotePrincipal = Mockito.mock(classOf[Principal]) + Mockito.when(peerCertificate.getSubjectDN).thenReturn(remotePrincipal) + Mockito.when(remotePrincipal.getName).thenReturn("remoteprincipal") + + registry + .connectionRegistry(addr).register(new ClientConnection { + override def remoteAddress: SocketAddress = remoteSocketAddress + override def localAddress: SocketAddress = addr + override def onClose: Future[Unit] = ??? + override def close(deadline: Time): Future[Unit] = ??? + override def sslSessionInfo: SslSessionInfo = new SslSessionInfo { + override def usingSsl: Boolean = true + override def session: SSLSession = ??? + override def sessionId: String = "sessionid" + override def cipherSuite: String = "cipher?sweeeeet!" + override def localCertificates: Seq[X509Certificate] = ??? + override def peerCertificates: Seq[X509Certificate] = Seq(peerCertificate) + } + }) + } +} + +class AttachedClientsHandlerTest extends FunSuite { + import AttachedClientsHandlerTest._ + + private[this] def assertJsonResponse(actualResponse: String, expectedResponse: String) = { + val actual = stripWhitespace(actualResponse) + val expected = stripWhitespace(expectedResponse) + assert(actual == expected) + } + + private[this] def stripWhitespace(string: String): String = + string.filter { case c => !c.isWhitespace } + + test("initial state") { + val registry = new ServerRegistry() + val handler = new AttachedClientsHandler(registry) + val response = await(handler(Request("/connections/"))) + val Buf.Utf8(content) = response.content + assertJsonResponse(content, """{"servers":[]}""") + } + + test("add a client connection") { + + val registry = new ServerRegistry() + registerServer(registry, InetSocketAddress.createUnresolved("/127.0.0.1", 8080), "server0") + + val handler = new AttachedClientsHandler(registry) + val response = await(handler(Request("/connections/"))) + + assert(response.status == Status.Ok) + assertJsonResponse( + response.contentString, + """ + |{ + | "servers": [ + | { + | "address": "127.0.0.1:8080", + | "clients": [ + | { + | "address": "127.0.0.1:9090", + | "ssl": { + | "session_id": "sessionid", + | "cipher_suite": "cipher?sweeeeet!", + | "peer_certificate": { + | "common_name": "remoteprincipal" + | } + | } + | } + | ] + | } + | ] + |} + """.stripMargin + ) + } +} From 8e5de6fadd9b7da6d0c27247626be8fa683eee1e Mon Sep 17 00:00:00 2001 From: Ruben Oanta Date: Wed, 17 Jul 2019 18:47:07 +0000 Subject: [PATCH 077/409] Twitter-oss: Prepare OSS libraries for release 19.7.0 Problem We want to release the next versions of our Twitter OSS libraries 19.7.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D342141 --- CHANGELOG.rst | 3 +++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 838e5ce4..7c1ea6c6 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +19.7.0 +------ + Changes ~~~~~~~ diff --git a/README.md b/README.md index dfacefc9..8bda9c7c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 70cdfd37..106cf975 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.7.0-SNAPSHOT" +val releaseVersion = "19.7.0" val jacksonVersion = "2.9.8" val jacksonLibs = Seq( From 8119791cf38520f20fc02bdd2d893459792b6b42 Mon Sep 17 00:00:00 2001 From: Dorothy Ordogh Date: Fri, 19 Jul 2019 01:54:55 +0000 Subject: [PATCH 078/409] Twitter-oss: update OSS libraries post-release to 19.8.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries: 19.8.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D342926 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8bda9c7c..dfacefc9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 106cf975..9a204bb3 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.7.0" +val releaseVersion = "19.8.0-SNAPSHOT" val jacksonVersion = "2.9.8" val jacksonLibs = Seq( From 5c97f5c6bbc4556fcb853b35b47db7d9a431c2c0 Mon Sep 17 00:00:00 2001 From: Matt Dannenberg Date: Fri, 19 Jul 2019 15:33:34 +0000 Subject: [PATCH 079/409] twitter-server: add histogram support to metric metadata endpoint prototype Problem Histograms are managed outside of StatsReceiver and as such did not end up in the initial metric metadata endpoint. Solution Pass histogram data into the MetricTypeQueryHandler so that it can report metadata info for histograms as well. Result Histograms are now listed in the metric metadata endpoint. JIRA Issues: CSL-8341 Differential Revision: https://phabricator.twitter.biz/D339684 --- .../main/scala/com/twitter/server/Admin.scala | 59 +++++++------- .../handler/MetricTypeQueryHandler.scala | 70 ++++++++++------ .../handler/MetricTypeQueryHandlerTest.scala | 80 +++++++++++++++---- 3 files changed, 139 insertions(+), 70 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/Admin.scala b/server/src/main/scala/com/twitter/server/Admin.scala index 735dbe6e..690315aa 100644 --- a/server/src/main/scala/com/twitter/server/Admin.scala +++ b/server/src/main/scala/com/twitter/server/Admin.scala @@ -186,13 +186,6 @@ trait Admin { self: App with AdminHttpServer with Stats => group = Some(Grouping.Metrics), includeInIndex = true ), - Route( - path = "/admin/exp/metric_metadata", - handler = new MetricTypeQueryHandler, - alias = "Metric Metadata", - group = Some(Grouping.Metrics), - includeInIndex = false - ), Route( path = Path.Clients, handler = new ClientRegistryHandler(Path.Clients), @@ -277,28 +270,40 @@ trait Admin { self: App with AdminHttpServer with Stats => val histos = DelegatingStatsReceiver .all(statsReceiver) .collect { case histo: WithHistogramDetails => histo } + val aggregate = if (histos.nonEmpty) Some(AggregateWithHistogramDetails(histos)) else None colocatedRoutes ++ standardRoutes ++ { - if (histos.nonEmpty) { - val aggregate = AggregateWithHistogramDetails(histos) - - val histogramHandler = new HistogramQueryHandler(aggregate) - Seq( - Route( - path = "/admin/histograms", - handler = histogramHandler, - alias = "Histograms", - group = Some(Grouping.Metrics), - includeInIndex = true - ), - Route( - path = "/admin/histograms.json", - handler = histogramHandler, - alias = "/admin/histograms.json", - group = Some(Grouping.Metrics), - includeInIndex = false + aggregate match { + case Some(details) => { + val histogramHandler = new HistogramQueryHandler(details) + Seq( + Route( + path = "/admin/histograms", + handler = histogramHandler, + alias = "Histograms", + group = Some(Grouping.Metrics), + includeInIndex = true + ), + Route( + path = "/admin/histograms.json", + handler = histogramHandler, + alias = "/admin/histograms.json", + group = Some(Grouping.Metrics), + includeInIndex = false + ) ) + } + case None => Nil + } + } ++ { + Seq( + Route( + path = "/admin/exp/metric_metadata", + handler = new MetricTypeQueryHandler(details = aggregate), + alias = "Metric Metadata", + group = Some(Grouping.Metrics), + includeInIndex = false ) - } else Nil + ) } } -} \ No newline at end of file +} diff --git a/server/src/main/scala/com/twitter/server/handler/MetricTypeQueryHandler.scala b/server/src/main/scala/com/twitter/server/handler/MetricTypeQueryHandler.scala index 6d2d5001..d54f10cb 100644 --- a/server/src/main/scala/com/twitter/server/handler/MetricTypeQueryHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/MetricTypeQueryHandler.scala @@ -2,6 +2,7 @@ package com.twitter.server.handler import com.twitter.finagle.Service import com.twitter.finagle.http.{MediaType, Request, Response} +import com.twitter.finagle.stats.{HistogramDetail, WithHistogramDetails} import com.twitter.io.Buf import com.twitter.server.util.HttpUtils.{newResponse, parse} import com.twitter.server.util.{JsonConverter, MetricSource} @@ -12,10 +13,12 @@ import com.twitter.util.Future * json encoded metrics with their types and an indicator of whether or not counters are latched. * * This is a temporary endpoint which will be replaced by a more fleshed out metadata endpoint. - * @note Histograms are not yet supported. + * + * @note When passing an explicit histogram metric via ?m=, users must provide the raw histogram + * name, no percentile (eg, .p99) appended. * * Example Request: - * http://$HOST:$PORT/admin/exp/metric_metadata?m=srv/http/requests&m=srv/http/pending + * http://$HOST:$PORT/admin/exp/metric_metadata?m=srv/http/requests&m=srv/http/pending&m=srv/mux/framer/write_stream_bytes * * Response: * { @@ -28,42 +31,57 @@ import com.twitter.util.Future * { * "name" : "srv/http/pending", * "kind" : "gauge" + * }, + * { + * "name" : "srv/mux/framer/write_stream_bytes", + * "kind" : "histogram" * } * ] * } */ -class MetricTypeQueryHandler(source: MetricSource = new MetricSource) +class MetricTypeQueryHandler( + source: MetricSource = new MetricSource, + details: Option[WithHistogramDetails] = None) extends Service[Request, Response] { + private[this] def histograms: Map[String, HistogramDetail] = + details match { + case Some(histos) => histos.histogramDetails + case None => Map.empty + } + private[this] def query(keys: Seq[String]) = for (k <- keys; e <- source.getType(k)) yield e + private[this] def getHistos(): Iterable[MetricSource.MetricTypeInfo] = { + histograms.map { + case (name, _) => MetricSource.MetricTypeInfo(name, "histogram") + } + } + + private[this] def queryHistos(keys: Set[String]): Iterable[MetricSource.MetricTypeInfo] = { + histograms.filterKeys(keys.contains).map { + case (name, _) => MetricSource.MetricTypeInfo(name, "histogram") + } + } + def apply(req: Request): Future[Response] = { val (_, params) = parse(req.uri) val latched = source.hasLatchedCounters - params.getOrElse("m", Nil) match { - case Nil => - newResponse( - contentType = MediaType.JsonUtf8, - content = Buf.Utf8( - JsonConverter.writeToString( - Map( - "latched" -> latched, - "metrics" -> source.typeMap - ))) - ) - - case someKeys => - newResponse( - contentType = MediaType.JsonUtf8, - content = Buf.Utf8( - JsonConverter.writeToString( - Map( - "latched" -> latched, - "metrics" -> query(someKeys) - ))) - ) + val metrics = params.get("m") match { + case None => source.typeMap ++ getHistos() + case Some(keysParam) => query(keysParam) ++ queryHistos(keysParam.toSet) } + + newResponse( + contentType = MediaType.JsonUtf8, + content = Buf.Utf8( + JsonConverter.writeToString( + Map( + "latched" -> latched, + "metrics" -> metrics + ))) + ) } -} \ No newline at end of file +} diff --git a/server/src/test/scala/com/twitter/server/handler/MetricTypeQueryHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/MetricTypeQueryHandlerTest.scala index 57a55137..48dbc038 100644 --- a/server/src/test/scala/com/twitter/server/handler/MetricTypeQueryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/MetricTypeQueryHandlerTest.scala @@ -1,7 +1,7 @@ package com.twitter.server.handler import com.twitter.finagle.http.Request -import com.twitter.finagle.stats.{StatEntry, StatsRegistry} +import com.twitter.finagle.stats.{InMemoryStatsReceiver, StatEntry, StatsRegistry} import com.twitter.server.util.MetricSource import com.twitter.util.{Await, Duration} import java.util.concurrent.TimeUnit @@ -14,18 +14,12 @@ class MetricTypeQueryHandlerTest extends FunSuite { trait UnlatchedRegistry extends StatsRegistry { val latched = false - val stats = mutable.Map( - "foo" -> TrialStat(3, 4, "counter"), - "bar" -> TrialStat(3, 4, "gauge"), - "baz" -> TrialStat(3, 4, "histogram")) + val stats = mutable.Map("foo" -> TrialStat(3, 4, "counter"), "bar" -> TrialStat(3, 4, "gauge")) } trait LatchedRegistry extends StatsRegistry { val latched = true - val stats = mutable.Map( - "foo" -> TrialStat(3, 4, "counter"), - "bar" -> TrialStat(3, 4, "gauge"), - "baz" -> TrialStat(3, 4, "histogram")) + val stats = mutable.Map("foo" -> TrialStat(3, 4, "counter"), "bar" -> TrialStat(3, 4, "gauge")) } val latchedStatsRegistry = new LatchedRegistry() { @@ -48,8 +42,17 @@ class MetricTypeQueryHandlerTest extends FunSuite { } ) - private[this] val latchedHandler = new MetricTypeQueryHandler(latchedMetricSource) - private[this] val unlatchedHandler = new MetricTypeQueryHandler(unlatchedMetricSource) + // generate artificial WithHistogramDetails as per HistogramQueryHandlerTest. + val histos = { + val sr = new InMemoryStatsReceiver + val myStat = sr.stat("baz") + sr + } + + private[this] val latchedHandler = + new MetricTypeQueryHandler(latchedMetricSource, details = Some(histos)) + private[this] val unlatchedHandler = + new MetricTypeQueryHandler(unlatchedMetricSource, details = Some(histos)) val typeRequestNoArg = Request("http://$HOST:$PORT/admin/exp/metric_metadata") @@ -57,20 +60,25 @@ class MetricTypeQueryHandlerTest extends FunSuite { val typeRequestWithManyArgs = Request("http://$HOST:$PORT/admin/exp/metric_metadata?m=foo&m=bar") + val typeRequestWithHisto = Request("http://$HOST:$PORT/admin/exp/metric_metadata?m=baz") + + val typeRequestWithHistoAndNon = Request( + "http://$HOST:$PORT/admin/exp/metric_metadata?m=foo&m=baz") + val responseToNoArg = """ | "metrics" : [ | { - | "name" : "baz", - | "kind" : "histogram" - | }, - | { | "name" : "foo", | "kind" : "counter" | }, | { | "name" : "bar", | "kind" : "gauge" + | }, + | { + | "name" : "baz", + | "kind" : "histogram" | } | ] | } @@ -102,6 +110,32 @@ class MetricTypeQueryHandlerTest extends FunSuite { | } """.stripMargin + val responseToHisto = + """ + | "metrics" : [ + | { + | "name" : "baz", + | "kind" : "histogram" + | } + | ] + | } + """.stripMargin + + val responseToHistoAndNon = + """ + | "metrics" : [ + | { + | "name" : "foo", + | "kind" : "counter" + | }, + | { + | "name" : "baz", + | "kind" : "histogram" + | } + | ] + | } + """.stripMargin + val testNameStart = "MetricTypeQueryHandler generates reasonable json for " val timeout = Duration(5, TimeUnit.SECONDS) @@ -123,6 +157,14 @@ class MetricTypeQueryHandlerTest extends FunSuite { testCase(latched, request, testNameStart + "a single requested metric", responseToAnArg) } else if (request == typeRequestWithManyArgs) { testCase(latched, request, testNameStart + "requested subset of metrics", responseToManyArgs) + } else if (request == typeRequestWithHisto) { + testCase(latched, request, testNameStart + "a single requested histogram", responseToHisto) + } else if (request == typeRequestWithHistoAndNon) { + testCase( + latched, + request, + testNameStart + "requested subset of metrics with a histogram", + responseToHistoAndNon) } } @@ -163,8 +205,12 @@ class MetricTypeQueryHandlerTest extends FunSuite { testCase(true, typeRequestNoArg), testCase(true, typeRequestWithManyArgs), testCase(true, typeRequestWithAnArg), + testCase(true, typeRequestWithHisto), + testCase(true, typeRequestWithHistoAndNon), testCase(false, typeRequestNoArg), testCase(false, typeRequestWithManyArgs), - testCase(false, typeRequestWithAnArg) + testCase(false, typeRequestWithAnArg), + testCase(false, typeRequestWithHisto), + testCase(false, typeRequestWithHistoAndNon) ) -} \ No newline at end of file +} From fb6ec3bbf727debce37b6953de72caca57fbffd9 Mon Sep 17 00:00:00 2001 From: Ruben Oanta Date: Thu, 25 Jul 2019 17:30:42 +0000 Subject: [PATCH 080/409] twitter-server: admin pages script order Problem We've empirically observed server instances that aren't able to render the charts in /admin/metrics. The current working theory is that this is a dependency issue in which google charts are loaded and the respective callback is dispatched before `chart-renderer.js` is loaded. Solution Load `chart-renderer.js` before trying to load the google charts lib. `ChartRenderer` depends on the the google charts libs, but it's only used after the libs have successfully loaded. Differential Revision: https://phabricator.twitter.biz/D345063 --- .../scala/com/twitter/server/handler/MetricQueryHandler.scala | 2 +- .../com/twitter/server/handler/ServerRegistryHandler.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/handler/MetricQueryHandler.scala b/server/src/main/scala/com/twitter/server/handler/MetricQueryHandler.scala index 6cd9585a..0f15209e 100644 --- a/server/src/main/scala/com/twitter/server/handler/MetricQueryHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/MetricQueryHandler.scala @@ -11,8 +11,8 @@ import com.twitter.util.Future private object MetricQueryHandler { def render(title: String, keys: Set[String]): String = s""" - +
      diff --git a/server/src/main/scala/com/twitter/server/handler/ServerRegistryHandler.scala b/server/src/main/scala/com/twitter/server/handler/ServerRegistryHandler.scala index a4ea8054..74e59100 100644 --- a/server/src/main/scala/com/twitter/server/handler/ServerRegistryHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/ServerRegistryHandler.scala @@ -14,8 +14,8 @@ import com.twitter.util.Future private object ServerRegistryHandler { def render(servers: Seq[(String, StackRegistry.Entry)]): String = s""" - +
    ${escapeHtml(updateMsg)}
    Daemon + data-content="See Thread.isDaemon Javadocs for details">
    - | - | - | - | - | - | - | - | - | - | - |
    Label${escapeHtml(md.label)}
    Balancer Class${escapeHtml(md.balancerClass)}
    Status${escapeHtml(md.status)}
    Number Nodes Available${md.numAvailable}
    Number Nodes Busy${md.numBusy}
    Number Nodes Closed${md.numClosed}
    Total pending requests${md.totalPending}
    Total load${md.totalLoad}
    Size${md.size}
    More details
    - """.stripMargin - } - - s""" - |
    - |
    - | - |

    Load Balancer

    - | $details - |
    - |
    - """.stripMargin + def getBalancers(filter: Option[String]): Seq[Metadata] = filter match { + case None => BalancerRegistry.get.allMetadata + case Some(label) => BalancerRegistry.get.allMetadata.filter(_.label == label) } + + def getBalancer(filter: Option[String]): Option[Metadata] = getBalancers(filter).headOption + } diff --git a/server/src/main/scala/com/twitter/server/view/BalancerHtmlView.scala b/server/src/main/scala/com/twitter/server/view/BalancerHtmlView.scala new file mode 100644 index 00000000..18640238 --- /dev/null +++ b/server/src/main/scala/com/twitter/server/view/BalancerHtmlView.scala @@ -0,0 +1,42 @@ +package com.twitter.server.view + +import com.twitter.finagle.loadbalancer.Metadata +import com.twitter.server.util.HtmlUtils.escapeHtml + +private[server] class BalancerHtmlView(balancer: Option[Metadata], routePath: String) extends View { + + private[view] def renderNoBalancerDetails: String = + "Load balancer not found in registry." + + private[view] def renderBalancerDetails(md: Metadata): String = + s""" + | + | + | + | + | + | + | + | + | + | + | + |
    Label${escapeHtml(md.label)}
    Balancer Class${escapeHtml(md.balancerClass)}
    Status${escapeHtml(md.status)}
    Number Nodes Available${md.numAvailable}
    Number Nodes Busy${md.numBusy}
    Number Nodes Closed${md.numClosed}
    Total pending requests${md.totalPending}
    Total load${md.totalLoad}
    Size${md.size}
    More details
    + """.stripMargin + + private[view] def renderDetails: String = balancer match { + case None => renderNoBalancerDetails + case Some(metadata) => renderBalancerDetails(metadata) + } + + def render: String = + s""" + |
    + |
    + | + |

    Load Balancer

    + | ${renderDetails} + |
    + |
    + """.stripMargin +} diff --git a/server/src/main/scala/com/twitter/server/view/BalancersJsonView.scala b/server/src/main/scala/com/twitter/server/view/BalancersJsonView.scala new file mode 100644 index 00000000..cc3dcbf3 --- /dev/null +++ b/server/src/main/scala/com/twitter/server/view/BalancersJsonView.scala @@ -0,0 +1,30 @@ +package com.twitter.server.view + +import com.twitter.finagle.loadbalancer.Metadata +import com.twitter.server.util.JsonConverter + +private[server] class BalancersJsonView(balancers: Seq[Metadata]) extends View { + + private[view] def renderBalancer(balancer: Metadata): Map[String, Object] = + Map( + "label" -> balancer.label, + "info" -> Map( + "balancer_class" -> balancer.balancerClass, + "status" -> balancer.status, + "number_available" -> balancer.numAvailable, + "number_busy" -> balancer.numBusy, + "number_closed" -> balancer.numClosed, + "total_pending" -> balancer.totalPending, + "total_load" -> balancer.totalLoad, + "size" -> balancer.size, + "additional_info" -> balancer.additionalInfo + ) + ) + + def render: String = { + val clients = balancers.map(renderBalancer) + val asMap: Map[String, Object] = Map("clients" -> clients) + JsonConverter.writeToString(asMap) + } + +} From ac53360007b53dd9993ccb1eaffa89fb51fcfdd5 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Mon, 4 Nov 2019 22:00:27 +0000 Subject: [PATCH 109/409] util-core: Pipe handles copying Problem/Solution: `copy` methods that copy from Readers to Writers should not be hosted under Reader. they are equally related to both parties. Moving them to `Pipe`. Result: Scala 2.12 supports Java calling companion object methods directly without many `$` and no users are using `copy`. I don't feel we should add Pipes.java for these two method temporarily until we drop 2.11 support. Having them still under Readers.java before removal. JIRA Issues: CSL-6756 Differential Revision: https://phabricator.twitter.biz/D393650 --- server/src/main/scala/com/twitter/server/view/IndexView.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/view/IndexView.scala b/server/src/main/scala/com/twitter/server/view/IndexView.scala index 9fee4a79..dd6d921b 100644 --- a/server/src/main/scala/com/twitter/server/view/IndexView.scala +++ b/server/src/main/scala/com/twitter/server/view/IndexView.scala @@ -5,7 +5,7 @@ import com.twitter.finagle.http.Method import com.twitter.finagle.http.Method.{Get, Post} import com.twitter.finagle.{Service, SimpleFilter} import com.twitter.finagle.http.{Request, Response} -import com.twitter.io.{Buf, BufReader, Reader} +import com.twitter.io.{Buf, BufReader, Reader, Pipe} import com.twitter.server.util.HtmlUtils.escapeHtml import com.twitter.server.util.HttpUtils.{expectsHtml, newResponse} import com.twitter.util.Future @@ -148,7 +148,7 @@ class IndexView(title: String, uri: String, index: () => Seq[IndexView.Entry]) response.contentType = "text/html;charset=UTF-8" response.setChunked(true) val reader = render(title, uri, index().sorted, res.reader) - Reader.copy(reader, response.writer) ensure response.writer.close() + Pipe.copy(reader, response.writer) ensure response.writer.close() Future.value(response) case res => From a48eb8bfc819ed9e8adb1c4feb9a95e8da25fe4e Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Tue, 5 Nov 2019 17:56:28 +0000 Subject: [PATCH 110/409] twitter-oss: Prepare OSS Libraries for Release 19.11.0 Problem We want to release the next versions of our Twitter OSS libraries 19.11.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-9036 Differential Revision: https://phabricator.twitter.biz/D395427 --- CHANGELOG.rst | 3 +++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 12f66fc9..60c3680d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +19.11.0 +------- + * Add initial support for JDK 11 compatibility. ``PHAB_ID=D365075`` * The endpoints section of the clients page has been fixed diff --git a/README.md b/README.md index dfacefc9..8bda9c7c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 5e587096..4541edd0 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.11.0-SNAPSHOT" +val releaseVersion = "19.11.0" val jacksonVersion = "2.9.9" val jacksonLibs = Seq( From db4d9631db24d58b733a12590c62b52c328060fc Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Wed, 6 Nov 2019 18:07:51 +0000 Subject: [PATCH 111/409] twitter-oss: Update OSS Libraries Post-Release to 19.12.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 19.12.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. JIRA Issues: CSL-9036 Differential Revision: https://phabricator.twitter.biz/D396071 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8bda9c7c..dfacefc9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 4541edd0..2554e887 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.11.0" +val releaseVersion = "19.12.0-SNAPSHOT" val jacksonVersion = "2.9.9" val jacksonLibs = Seq( From f8ad2e5a52a5ff99e54b65cd580ee2729aa532bf Mon Sep 17 00:00:00 2001 From: Hamdi Allam Date: Thu, 7 Nov 2019 23:57:27 +0000 Subject: [PATCH 112/409] twitter-server: move IsolateFilter into the filter package Problem AdminHttpServer.Route mixes the route definition with its companion object and also defines a filter to offload work into another thread pool. The `IsolateFilter` class name also does not provide a clear indication as to what it does. Solution Let's define a filter package containing the filter and rename IsolateFilter to AdminThreadPoolFilter JIRA Issues: CSL-9013 Differential Revision: https://phabricator.twitter.biz/D392136 --- .../main/scala/com/twitter/server/Admin.scala | 3 +- .../com/twitter/server/AdminHttpServer.scala | 70 ++++++------------- .../filters/AdminThreadPoolFilter.scala | 33 +++++++++ .../filters/AdminThreadPoolFilterTest.scala | 30 ++++++++ 4 files changed, 86 insertions(+), 50 deletions(-) create mode 100644 server/src/main/scala/com/twitter/server/filters/AdminThreadPoolFilter.scala create mode 100644 server/src/test/scala/com/twitter/server/filters/AdminThreadPoolFilterTest.scala diff --git a/server/src/main/scala/com/twitter/server/Admin.scala b/server/src/main/scala/com/twitter/server/Admin.scala index fe070859..a81eaec0 100644 --- a/server/src/main/scala/com/twitter/server/Admin.scala +++ b/server/src/main/scala/com/twitter/server/Admin.scala @@ -9,6 +9,7 @@ import com.twitter.finagle.stats.{ WithHistogramDetails } import com.twitter.server.AdminHttpServer.Route +import com.twitter.server.filters.AdminThreadPoolFilter import com.twitter.server.handler._ import com.twitter.server.view._ @@ -238,7 +239,7 @@ object Admin { group = Some(Grouping.Utilities), includeInIndex = true ) - ).map(AdminHttpServer.Route.isolate) + ).map(AdminThreadPoolFilter.isolateRoute) // If histograms are available, add an additional endpoint val histos = DelegatingStatsReceiver diff --git a/server/src/main/scala/com/twitter/server/AdminHttpServer.scala b/server/src/main/scala/com/twitter/server/AdminHttpServer.scala index 974454f0..87de8f16 100644 --- a/server/src/main/scala/com/twitter/server/AdminHttpServer.scala +++ b/server/src/main/scala/com/twitter/server/AdminHttpServer.scala @@ -1,32 +1,35 @@ package com.twitter.server import com.twitter.app.App -import com.twitter.concurrent.NamedPoolThreadFactory import com.twitter.finagle.client.ClientRegistry import com.twitter.finagle.filter.ServerAdmissionControl -import com.twitter.finagle.http.Method.Get -import com.twitter.finagle.http.{HttpMuxer, Method, Request, Response} +import com.twitter.finagle.http.{HttpMuxer, Method, Request, Response, Route => HttpRoute} import com.twitter.finagle.server.ServerRegistry import com.twitter.finagle.stats.NullStatsReceiver import com.twitter.finagle.tracing.NullTracer import com.twitter.finagle.util.LoadService -import com.twitter.finagle.{Http, ListeningServer, NullServer, Service, SimpleFilter, http} +import com.twitter.finagle.{Http, ListeningServer, NullServer, Service} import com.twitter.server.Admin.Grouping +import com.twitter.server.filters.AdminThreadPoolFilter import com.twitter.server.handler.{AdminHttpMuxHandler, NoLoggingHandler} import com.twitter.server.lint.LoggingRules import com.twitter.server.util.HttpUtils import com.twitter.server.view.{IndexView, NotFoundView} import com.twitter.util.lint.GlobalRules import com.twitter.util.registry.Library -import com.twitter.util.{ExecutorServiceFuturePool, Future, FuturePool, Monitor, Time} +import com.twitter.util.{Future, Monitor, Time} import java.net.InetSocketAddress -import java.util.concurrent.Executors import org.slf4j.LoggerFactory import scala.language.reflectiveCalls import com.twitter.app.Flag object AdminHttpServer { + /** + * The name used for the finagle server. + */ + val ServerName = "adminhttp" + /** * Represents an element which can be routed to via the admin http server. * @@ -55,42 +58,17 @@ object AdminHttpServer { alias: String, group: Option[String], includeInIndex: Boolean, - method: Method = Get) + method: Method = Method.Get) object Route { - - /** - * A filter which forces an [[Http]] [[Service]] to be handled outside the - * global default worker pool. - */ - private[this] val IsolateFilter: SimpleFilter[Request, Response] = - new SimpleFilter[Request, Response] { - def apply(request: Request, service: Service[Request, Response]): Future[Response] = - Pool(service(request)).flatten - override def toString: String = s"${Route.getClass.getName}IsolateFilter" - } - - private[this] lazy val Pool: FuturePool = - new ExecutorServiceFuturePool( - Executors - .newCachedThreadPool(new NamedPoolThreadFactory("AdminFuturePool", makeDaemons = true)) - ) { override def toString: String = "Route.Pool" } - - /** - * Force the [[Route]] `r` to be handled outside the global default worker pool. - */ - def isolate(r: Route): Route = - r.copy(handler = IsolateFilter.andThen(r.handler)) - - /** - * Force the [[Service[Request, Response]] `s` to be handled outside the global default worker pool. - */ + // backwards compatibility + def isolate(r: Route): Route = AdminThreadPoolFilter.isolateRoute(r) def isolate(s: Service[Request, Response]): Service[Request, Response] = - IsolateFilter.andThen(s) + AdminThreadPoolFilter.isolateService(s) - def from(route: http.Route): Route = route.index match { + def from(route: HttpRoute): Route = route.index match { case Some(index) => - mkRoute( + Route( path = index.path.getOrElse(route.pattern), handler = route.handler, alias = index.alias, @@ -99,7 +77,7 @@ object AdminHttpServer { method = index.method ) case None => - mkRoute( + Route( path = route.pattern, handler = route.handler, alias = route.pattern, @@ -118,16 +96,10 @@ object AdminHttpServer { alias: String, group: Option[String], includeInIndex: Boolean, - method: Method = Get + method: Method = Method.Get ): Route = { Route(path, handler, alias, group, includeInIndex, method) } - - /** - * The name used for the finagle server. - */ - val ServerName = "adminhttp" - } trait AdminHttpServer { self: App with Stats => @@ -159,7 +131,7 @@ trait AdminHttpServer { self: App with Stats => private[this] val loadServiceRoutes: Seq[Route] = LoadService[AdminHttpMuxHandler]() .map(handler => Route.from(handler.route)) - .map(Route.isolate) + .map(AdminThreadPoolFilter.isolateRoute) private[this] var allRoutes: Seq[Route] = loadServiceRoutes @@ -211,14 +183,14 @@ trait AdminHttpServer { self: App with Stats => allRoutes = allRoutes ++ Seq( - mkRoute( + Route( path = "/admin/logging", handler = new NoLoggingHandler, alias = "Logging", group = Some(Grouping.Utilities), includeInIndex = true ) - ).map(Route.isolate) + ).map(AdminThreadPoolFilter.isolateRoute) } } @@ -284,7 +256,7 @@ trait AdminHttpServer { self: App with Stats => .toSeq } - private[this] def routeToIndexLink(route: AdminHttpServer.Route): IndexView.Link = + private[this] def routeToIndexLink(route: Route): IndexView.Link = IndexView.Link(route.alias, route.path, route.method) private[this] def startServer(): Unit = { diff --git a/server/src/main/scala/com/twitter/server/filters/AdminThreadPoolFilter.scala b/server/src/main/scala/com/twitter/server/filters/AdminThreadPoolFilter.scala new file mode 100644 index 00000000..626dc0c4 --- /dev/null +++ b/server/src/main/scala/com/twitter/server/filters/AdminThreadPoolFilter.scala @@ -0,0 +1,33 @@ +package com.twitter.server.filters + +import com.twitter.concurrent.NamedPoolThreadFactory +import com.twitter.finagle.{Service} +import com.twitter.finagle.filter.OffloadFilter +import com.twitter.finagle.http.{Request, Response} +import com.twitter.finagle.filter.OffloadFilter +import com.twitter.server.AdminHttpServer.Route +import com.twitter.util.{ExecutorServiceFuturePool, FuturePool} +import java.util.concurrent.Executors + +object AdminThreadPoolFilter { + // scoped for testing + private[filters] lazy val Pool: FuturePool = + new ExecutorServiceFuturePool( + Executors + .newCachedThreadPool(new NamedPoolThreadFactory("AdminFuturePool", makeDaemons = true)) + ) { + override def toString: String = "Admin.FuturePool" + } + + /** + * Force the [[Route]] `r` to be handled by a dedicated admin thread pool. + */ + def isolateRoute(r: Route): Route = + r.copy(handler = isolateService(r.handler)) + + /** + * Force the [[Service[Request, Response]] `s` to be handled by a dedicated admin thread pool. + */ + def isolateService(s: Service[Request, Response]): Service[Request, Response] = + (new OffloadFilter.Server(Pool)).andThen(s) +} diff --git a/server/src/test/scala/com/twitter/server/filters/AdminThreadPoolFilterTest.scala b/server/src/test/scala/com/twitter/server/filters/AdminThreadPoolFilterTest.scala new file mode 100644 index 00000000..508c58a9 --- /dev/null +++ b/server/src/test/scala/com/twitter/server/filters/AdminThreadPoolFilterTest.scala @@ -0,0 +1,30 @@ +package com.twitter.server.filters + +import com.twitter.conversions.DurationOps._ +import com.twitter.finagle.Service +import com.twitter.finagle.http.{Request, Response} +import com.twitter.util.{Await, Future} +import org.scalatest.FunSuite + +class AdminThreadPoolFilterTest extends FunSuite { + test("work done in the dedicated admin thread pool") { + @volatile var workingThreadId: Long = -1 + + val svc = new Service[Request, Response] { + def apply(request: Request): Future[Response] = { + workingThreadId = Thread.currentThread().getId + + Future.value(Response()) + } + } + val s = AdminThreadPoolFilter.isolateService(svc) + + val caller = Thread.currentThread() + // will also set `workingThreadId` + Await.result(s(Request()), 5.seconds) + + assert(workingThreadId != -1) + assert(workingThreadId != caller.getId) + assert(AdminThreadPoolFilter.Pool.numCompletedTasks == 1) + } +} From b922ee4bcf303a40a76d7638f98fedaf2f0d8cc9 Mon Sep 17 00:00:00 2001 From: Eitan Adler Date: Tue, 12 Nov 2019 01:32:35 +0000 Subject: [PATCH 113/409] [logging] lift slf4j-log4j12 deps from libraries to binaries Problem slfj is a logging facade with several implementations - which may be loaded at runtime. It is generally incorrect to include an implementation in a library. Instead, include just the API in library code and allow applications to depend on a specific implementation in the binary Some libraries include logging implementations. Solution lift the implementation dependency from library to binary. Result a No-Op change Differential Revision: https://phabricator.twitter.biz/D398671 --- slf4j-log4j12/src/main/scala/BUILD | 53 +++++++++++++++--------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/slf4j-log4j12/src/main/scala/BUILD b/slf4j-log4j12/src/main/scala/BUILD index f575c444..d0f3acde 100644 --- a/slf4j-log4j12/src/main/scala/BUILD +++ b/slf4j-log4j12/src/main/scala/BUILD @@ -1,27 +1,28 @@ scala_library( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, - provides = scala_artifact( - org = "com.twitter", - name = "twitter-server-slf4j-log4j12", - repo = artifactory, - ), - scope = "forced", # Exposes a service-loaded class. - dependencies = [ - "3rdparty/jvm/log4j", - "3rdparty/jvm/org/slf4j:jcl-over-slf4j", - "3rdparty/jvm/org/slf4j:jul-to-slf4j", - "3rdparty/jvm/org/slf4j:slf4j-api", - "3rdparty/jvm/org/slf4j:slf4j-log4j12", - "finagle/finagle-base-http/src/main/scala", - "finagle/finagle-http/src/main/scala", - "twitter-server/server/src/main/scala", - "twitter-server/slf4j-log4j12/src/main/resources", - "util/util-slf4j-api/src/main/scala", - ], - exports = [ - "3rdparty/jvm/log4j", - "3rdparty/jvm/org/slf4j:slf4j-api", - "3rdparty/jvm/org/slf4j:slf4j-log4j12", - ], -) + sources = rglobs("*.scala"), + compiler_option_sets = {"fatal_warnings"}, + provides = scala_artifact( + org = "com.twitter", + name = "twitter-server-slf4j-log4j12", + repo = artifactory, + ), + scope = "forced", # Exposes a service-loaded class. + tags = ["logging_impl_check_whitelisted_target"], + dependencies = [ + "3rdparty/jvm/log4j", + "3rdparty/jvm/org/slf4j:jcl-over-slf4j", + "3rdparty/jvm/org/slf4j:jul-to-slf4j", + "3rdparty/jvm/org/slf4j:slf4j-api", + "3rdparty/jvm/org/slf4j:slf4j-log4j12", + "finagle/finagle-base-http/src/main/scala", + "finagle/finagle-http/src/main/scala", + "twitter-server/server/src/main/scala", + "twitter-server/slf4j-log4j12/src/main/resources", + "util/util-slf4j-api/src/main/scala", + ], + exports = [ + "3rdparty/jvm/log4j", + "3rdparty/jvm/org/slf4j:slf4j-api", + "3rdparty/jvm/org/slf4j:slf4j-log4j12", + ], +) \ No newline at end of file From b538228c4401842a76a758e038f4a2ed73e98f5d Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Tue, 12 Nov 2019 18:44:40 +0000 Subject: [PATCH 114/409] twitter-server: Remove Duplicate URI Parsing Problem / Solution finagle-http contains the `Uri` class which has the responsibility for parsing paths and query params from a URI. Twitter Server contains code which does roughly the same job. It should instead be using finagle-http's `Uri` class for this. Let's fix that. JIRA Issues: CSL-8797 Differential Revision: https://phabricator.twitter.biz/D398387 --- CHANGELOG.rst | 11 ++++++- .../handler/ClientRegistryHandler.scala | 8 ++--- .../server/handler/HeapResourceHandler.scala | 15 +++++---- .../handler/HistogramQueryHandler.scala | 20 ++++++----- .../server/handler/MetricQueryHandler.scala | 33 +++++++++---------- .../handler/MetricTypeQueryHandler.scala | 17 +++++----- .../handler/ProfileResourceHandler.scala | 11 ++++--- .../server/handler/ResourceHandler.scala | 8 ++--- .../handler/ServerRegistryHandler.scala | 8 ++--- .../server/handler/ShutdownHandler.scala | 10 +++--- .../server/handler/TracingHandler.scala | 11 ++++--- .../server/handler/TwitterHandler.scala | 6 +--- .../com/twitter/server/util/HttpUtils.scala | 8 ----- .../twitter/server/util/HttpUtilsTest.scala | 14 -------- .../handler/slf4j/jdk14/LoggingHandler.scala | 11 +++---- 15 files changed, 89 insertions(+), 102 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 60c3680d..b4105665 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,15 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +* Multiple changes have happened around query parameter retrieval in order + to remove duplicated functionality from Twitter Server that exists in + Finagle. Users are encouraged to use finagle-http's `Uri` class within their + own code to retrieve params. ``PHAB_ID=D398387`` + * The `parse` method of `HttpUtils` has been removed. + * The protected `getParams` method of `TwitterHandler` has been removed. + * The signature of the `getGraceParam` method of `ShutdownHandler` has + been changed to take a `Request`. + 19.11.0 ------- @@ -684,4 +693,4 @@ Breaking API Changes 1.0.1 ----- -- Initial Release \ No newline at end of file +- Initial Release diff --git a/server/src/main/scala/com/twitter/server/handler/ClientRegistryHandler.scala b/server/src/main/scala/com/twitter/server/handler/ClientRegistryHandler.scala index 3355d962..0144a708 100644 --- a/server/src/main/scala/com/twitter/server/handler/ClientRegistryHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/ClientRegistryHandler.scala @@ -2,12 +2,12 @@ package com.twitter.server.handler import com.twitter.finagle.Service import com.twitter.finagle.client.{ClientRegistry, EndpointRegistry} -import com.twitter.finagle.http.{Request, Response} +import com.twitter.finagle.http.{Request, Response, Uri} import com.twitter.finagle.util.StackRegistry import com.twitter.io.Buf import com.twitter.server.model.ClientProfile import com.twitter.server.util.HtmlUtils.escapeHtml -import com.twitter.server.util.HttpUtils.{parse, new404, newResponse} +import com.twitter.server.util.HttpUtils.{new404, newResponse} import com.twitter.server.util.MetricSource import com.twitter.server.view.{BalancerHtmlView, EndpointRegistryView, StackRegistryView} import com.twitter.util.Future @@ -120,8 +120,8 @@ class ClientRegistryHandler( }).toSeq def apply(req: Request): Future[Response] = { - val (path, _) = parse(req.uri) - path.stripPrefix(uriPrefix) match { + val uri = Uri.fromRequest(req) + uri.path.stripPrefix(uriPrefix) match { case idx @ ("index.html" | "index.htm" | "index.txt" | "clients") => val leastPerformant = clientProfiles.sorted(profileOrdering).take(4) val html = diff --git a/server/src/main/scala/com/twitter/server/handler/HeapResourceHandler.scala b/server/src/main/scala/com/twitter/server/handler/HeapResourceHandler.scala index 65453ca6..0f297768 100644 --- a/server/src/main/scala/com/twitter/server/handler/HeapResourceHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/HeapResourceHandler.scala @@ -1,11 +1,11 @@ package com.twitter.server.handler import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.http.{Request, Response, Status} +import com.twitter.finagle.http.{Request, Response, Status, Uri} import com.twitter.finagle.Service import com.twitter.io.Buf import com.twitter.jvm.Heapster -import com.twitter.server.util.HttpUtils.{newResponse, parse} +import com.twitter.server.util.HttpUtils.newResponse import com.twitter.util.{Duration, Future} import com.twitter.util.logging.Logger @@ -24,14 +24,15 @@ class HeapResourceHandler extends Service[Request, Response] { val heapster = Heapster.instance.get - val params = parse(req.uri)._2.foldLeft(Params(10.seconds, 10 << 19, forceGC = true)) { - case (parameters, ("pause", Seq(pauseVal))) => + val uri = Uri.fromRequest(req) + val params = uri.params.foldLeft(Params(10.seconds, 10 << 19, forceGC = true)) { + case (parameters, ("pause", pauseVal)) => parameters.copy(pause = pauseVal.toInt.seconds) - case (parameters, ("sample_period", Seq(sampleVal))) => + case (parameters, ("sample_period", sampleVal)) => parameters.copy(samplingPeriod = sampleVal.toInt) - case (parameters, ("force_gc", Seq("no"))) => + case (parameters, ("force_gc", "no")) => parameters.copy(forceGC = false) - case (parameters, ("force_gc", Seq("0"))) => + case (parameters, ("force_gc", "0")) => parameters.copy(forceGC = false) case (parameters, _) => parameters diff --git a/server/src/main/scala/com/twitter/server/handler/HistogramQueryHandler.scala b/server/src/main/scala/com/twitter/server/handler/HistogramQueryHandler.scala index 7e87c553..afe88e32 100644 --- a/server/src/main/scala/com/twitter/server/handler/HistogramQueryHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/HistogramQueryHandler.scala @@ -1,11 +1,11 @@ package com.twitter.server.handler import com.twitter.finagle.Service -import com.twitter.finagle.http.{Request, Response} +import com.twitter.finagle.http.{Request, Response, Uri} import com.twitter.finagle.stats.{BucketAndCount, HistogramDetail, WithHistogramDetails} import com.twitter.io.Buf import com.twitter.server.util.HtmlUtils.escapeHtml -import com.twitter.server.util.HttpUtils.{newResponse, parse} +import com.twitter.server.util.HttpUtils.newResponse import com.twitter.server.util.JsonConverter import com.twitter.util.Future @@ -331,7 +331,9 @@ private[server] class HistogramQueryHandler(details: WithHistogramDetails) * If `h` is not found, an empty JSON hash will be returned `{}`. */ def apply(req: Request): Future[Response] = { - val (path, params) = parse(req.uri) + val uri = Uri.fromRequest(req) + val path = uri.path + val params = uri.params path match { case "/admin/histograms.json" => @@ -339,7 +341,7 @@ private[server] class HistogramQueryHandler(details: WithHistogramDetails) newResponse(contentType = ContentTypeJson, content = Buf.Utf8(renderHistogramsJson)) } else { val summary: Option[Summary] = params.get("h") match { - case Some(Seq(histoName)) => generateSummary(histoName) + case Some(histoName) => generateSummary(histoName) case _ => None } val text: String = summary match { @@ -350,22 +352,22 @@ private[server] class HistogramQueryHandler(details: WithHistogramDetails) } case "/admin/histograms" => params.get("h") match { - case Some(Seq(query)) => + case Some(query) => params.get("fmt") match { - case Some(Seq("plot_pdf")) | Some(Seq("plot_cdf")) => + case Some("plot_pdf") | Some("plot_cdf") => htmlResponse(query) - case Some(Seq("raw")) => + case Some("raw") => jsonResponse(query, { counts: Seq[BucketAndCount] => deliverData(Map(query -> counts), identity) }) - case Some(Seq("pdf")) => + case Some("pdf") => jsonResponse(query, { counts: Seq[BucketAndCount] => deliverData(Map(query -> counts), x => pdf(x)) }) - case Some(Seq("cdf")) => + case Some("cdf") => jsonResponse(query, { counts: Seq[BucketAndCount] => deliverData(Map(query -> counts), x => cdf(x)) }) diff --git a/server/src/main/scala/com/twitter/server/handler/MetricQueryHandler.scala b/server/src/main/scala/com/twitter/server/handler/MetricQueryHandler.scala index 0f15209e..2007eb7b 100644 --- a/server/src/main/scala/com/twitter/server/handler/MetricQueryHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/MetricQueryHandler.scala @@ -1,10 +1,10 @@ package com.twitter.server.handler import com.twitter.finagle.Service -import com.twitter.finagle.http.{MediaType, Request, Response} +import com.twitter.finagle.http.{MediaType, Request, Response, Uri} import com.twitter.io.Buf import com.twitter.server.util.HtmlUtils.escapeHtml -import com.twitter.server.util.HttpUtils.{newResponse, parse} +import com.twitter.server.util.HttpUtils.newResponse import com.twitter.server.util.{JsonConverter, MetricSource} import com.twitter.util.Future @@ -35,24 +35,23 @@ class MetricQueryHandler(source: MetricSource = new MetricSource) extends Service[Request, Response] { import MetricQueryHandler._ - private[this] def query(keys: Seq[String]) = + private[this] def query(keys: Iterable[String]) = for (k <- keys; e <- source.get(k)) yield e def apply(req: Request): Future[Response] = { - val (_, params) = parse(req.uri) + val uri = Uri.fromRequest(req) - params.getOrElse("m", Nil) match { - case Nil => - newResponse( - contentType = "text/html;charset=UTF-8", - content = Buf.Utf8(render("Test", source.keySet)) - ) - - case someKeys => - newResponse( - contentType = MediaType.JsonUtf8, - content = Buf.Utf8(JsonConverter.writeToString(query(someKeys))) - ) + if (uri.params.contains("m")) { + val someKeys = uri.params.getAll("m") + newResponse( + contentType = MediaType.JsonUtf8, + content = Buf.Utf8(JsonConverter.writeToString(query(someKeys))) + ) + } else { + newResponse( + contentType = "text/html;charset=UTF-8", + content = Buf.Utf8(render("Test", source.keySet)) + ) } } -} \ No newline at end of file +} diff --git a/server/src/main/scala/com/twitter/server/handler/MetricTypeQueryHandler.scala b/server/src/main/scala/com/twitter/server/handler/MetricTypeQueryHandler.scala index d54f10cb..8ecb2d9f 100644 --- a/server/src/main/scala/com/twitter/server/handler/MetricTypeQueryHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/MetricTypeQueryHandler.scala @@ -1,10 +1,10 @@ package com.twitter.server.handler import com.twitter.finagle.Service -import com.twitter.finagle.http.{MediaType, Request, Response} +import com.twitter.finagle.http.{MediaType, Request, Response, Uri} import com.twitter.finagle.stats.{HistogramDetail, WithHistogramDetails} import com.twitter.io.Buf -import com.twitter.server.util.HttpUtils.{newResponse, parse} +import com.twitter.server.util.HttpUtils.newResponse import com.twitter.server.util.{JsonConverter, MetricSource} import com.twitter.util.Future @@ -50,7 +50,7 @@ class MetricTypeQueryHandler( case None => Map.empty } - private[this] def query(keys: Seq[String]) = + private[this] def query(keys: Iterable[String]) = for (k <- keys; e <- source.getType(k)) yield e private[this] def getHistos(): Iterable[MetricSource.MetricTypeInfo] = { @@ -66,13 +66,14 @@ class MetricTypeQueryHandler( } def apply(req: Request): Future[Response] = { - val (_, params) = parse(req.uri) + val uri = Uri.fromRequest(req) val latched = source.hasLatchedCounters - val metrics = params.get("m") match { - case None => source.typeMap ++ getHistos() - case Some(keysParam) => query(keysParam) ++ queryHistos(keysParam.toSet) - } + val keysParam = uri.params.getAll("m") + + val metrics = + if (keysParam.isEmpty) source.typeMap ++ getHistos() + else query(keysParam) ++ queryHistos(keysParam.toSet) newResponse( contentType = MediaType.JsonUtf8, diff --git a/server/src/main/scala/com/twitter/server/handler/ProfileResourceHandler.scala b/server/src/main/scala/com/twitter/server/handler/ProfileResourceHandler.scala index 318fbf65..f19b9986 100644 --- a/server/src/main/scala/com/twitter/server/handler/ProfileResourceHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/ProfileResourceHandler.scala @@ -1,11 +1,11 @@ package com.twitter.server.handler import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.http.{Request, Response, Status} +import com.twitter.finagle.http.{Request, Response, Status, Uri} import com.twitter.finagle.Service import com.twitter.io.Buf import com.twitter.jvm.CpuProfile -import com.twitter.server.util.HttpUtils.{newResponse, parse} +import com.twitter.server.util.HttpUtils.newResponse import com.twitter.util.{Duration, Future, Return, Throw} import java.io.ByteArrayOutputStream import com.twitter.util.logging.Logger @@ -16,10 +16,11 @@ class ProfileResourceHandler(which: Thread.State) extends Service[Request, Respo case class Params(pause: Duration, frequency: Int) def apply(req: Request): Future[Response] = { - val params = parse(req.uri)._2.foldLeft(Params(10.seconds, 100)) { - case (parameters, ("seconds", Seq(pauseVal))) => + val uri = Uri.fromRequest(req) + val params = uri.params.foldLeft(Params(10.seconds, 100)) { + case (parameters, ("seconds", pauseVal)) => parameters.copy(pause = pauseVal.toInt.seconds) - case (parameters, ("hz", Seq(hz))) => + case (parameters, ("hz", hz)) => parameters.copy(frequency = hz.toInt) case (parameters, _) => parameters diff --git a/server/src/main/scala/com/twitter/server/handler/ResourceHandler.scala b/server/src/main/scala/com/twitter/server/handler/ResourceHandler.scala index 981c80c9..c4a3a2b9 100644 --- a/server/src/main/scala/com/twitter/server/handler/ResourceHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/ResourceHandler.scala @@ -1,9 +1,9 @@ package com.twitter.server.handler import com.twitter.finagle.Service -import com.twitter.finagle.http.{Request, Response, Status} +import com.twitter.finagle.http.{Request, Response, Status, Uri} import com.twitter.io.Buf -import com.twitter.server.util.HttpUtils.{new404, newResponse, parse} +import com.twitter.server.util.HttpUtils.{new404, newResponse} import com.twitter.util.{Future, FuturePool, JavaSingleton} import java.io.{File, FileInputStream, InputStream} import java.nio.charset.{Charset, StandardCharsets} @@ -36,8 +36,8 @@ class ResourceHandler( } def apply(req: Request): Future[Response] = { - val (uri, _) = parse(req.uri) - val path = uri.stripPrefix(baseRequestPath) + val uri = Uri.fromRequest(req) + val path = uri.path.stripPrefix(baseRequestPath) if (path.contains("..")) return newResponse( diff --git a/server/src/main/scala/com/twitter/server/handler/ServerRegistryHandler.scala b/server/src/main/scala/com/twitter/server/handler/ServerRegistryHandler.scala index 74e59100..00b004fb 100644 --- a/server/src/main/scala/com/twitter/server/handler/ServerRegistryHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/ServerRegistryHandler.scala @@ -1,12 +1,12 @@ package com.twitter.server.handler import com.twitter.finagle.Service -import com.twitter.finagle.http.{Request, Response} +import com.twitter.finagle.http.{Request, Response, Uri} import com.twitter.finagle.server.ServerRegistry import com.twitter.finagle.util.StackRegistry import com.twitter.io.Buf import com.twitter.server.util.HtmlUtils.escapeHtml -import com.twitter.server.util.HttpUtils.{new404, newResponse, parse} +import com.twitter.server.util.HttpUtils.{new404, newResponse} import com.twitter.server.util.MetricSource import com.twitter.server.view.StackRegistryView import com.twitter.util.Future @@ -77,8 +77,8 @@ class ServerRegistryHandler( } def apply(req: Request): Future[Response] = { - val (path, _) = parse(req.uri) - path.stripPrefix(uriPrefix) match { + val uri = Uri.fromRequest(req) + uri.path.stripPrefix(uriPrefix) match { case idx @ ("index.html" | "index.htm" | "index.txt" | "servers") => val servers = (registry.registrants.flatMap { case e: StackRegistry.Entry if e.name.nonEmpty => diff --git a/server/src/main/scala/com/twitter/server/handler/ShutdownHandler.scala b/server/src/main/scala/com/twitter/server/handler/ShutdownHandler.scala index 8134a857..307c2b1d 100644 --- a/server/src/main/scala/com/twitter/server/handler/ShutdownHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/ShutdownHandler.scala @@ -1,22 +1,22 @@ package com.twitter.server.handler import com.twitter.app.App -import com.twitter.finagle.http.{Method, Request, Response, Status} +import com.twitter.finagle.http.{Method, Request, Response, Status, Uri} import com.twitter.finagle.Service import com.twitter.io.Buf -import com.twitter.server.util.HttpUtils.{parse, newOk, newResponse} +import com.twitter.server.util.HttpUtils.{newOk, newResponse} import com.twitter.util.{Duration, Future} import com.twitter.util.logging.Logger class ShutdownHandler(app: App) extends Service[Request, Response] { private[this] val log = Logger[ShutdownHandler] - protected def getGraceParam(uri: String): Option[String] = - parse(uri)._2.get("grace").flatMap(_.headOption) + protected def getGraceParam(request: Request): Option[String] = + Uri.fromRequest(request).params.get("grace") def apply(req: Request): Future[Response] = { if (req.method == Method.Post) { - val specifiedGracePeriod = getGraceParam(req.uri).map { d => + val specifiedGracePeriod = getGraceParam(req).map { d => try Duration.parse(d) catch { case e: NumberFormatException => diff --git a/server/src/main/scala/com/twitter/server/handler/TracingHandler.scala b/server/src/main/scala/com/twitter/server/handler/TracingHandler.scala index f4985aee..09725aa8 100644 --- a/server/src/main/scala/com/twitter/server/handler/TracingHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/TracingHandler.scala @@ -1,9 +1,9 @@ package com.twitter.server.handler import com.twitter.finagle.Service -import com.twitter.finagle.http.{Request, Response, Status} +import com.twitter.finagle.http.{Request, Response, Status, Uri} import com.twitter.io.Buf -import com.twitter.server.util.HttpUtils.{newResponse, parse} +import com.twitter.server.util.HttpUtils.newResponse import com.twitter.util.Future import com.twitter.util.logging.Logger @@ -32,7 +32,8 @@ class TracingHandler extends Service[Request, Response] { private[this] val log = Logger[TracingHandler] def apply(request: Request): Future[Response] = { - val (_, params) = parse(request.uri) + val uri = Uri.fromRequest(request) + val params = uri.params try { if (FinagleTracing.instance.isEmpty) @@ -53,10 +54,10 @@ class TracingHandler extends Service[Request, Response] { } val tracing = FinagleTracing.instance.get - val msg = if (params.getOrElse("enable", Seq()).headOption.equals(Some("true"))) { + val msg = if (params.getOrElse("enable", "") == "true") { tracing.enable() "Enabled Finagle tracing" - } else if (params.getOrElse("disable", Seq()).headOption.equals(Some("true"))) { + } else if (params.getOrElse("disable", "") == "true") { tracing.disable() "Disabling Finagle tracing" } else { diff --git a/server/src/main/scala/com/twitter/server/handler/TwitterHandler.scala b/server/src/main/scala/com/twitter/server/handler/TwitterHandler.scala index ada8a5da..3ff5b1c2 100644 --- a/server/src/main/scala/com/twitter/server/handler/TwitterHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/TwitterHandler.scala @@ -3,10 +3,9 @@ package com.twitter.server.handler import com.twitter.finagle.Service import com.twitter.finagle.http.{Request, Response, Status} import com.twitter.io.Buf -import com.twitter.server.util.HttpUtils.{newResponse, parse} +import com.twitter.server.util.HttpUtils.newResponse import com.twitter.util.Future import com.twitter.util.logging.Logger -import scala.collection.{Map, Seq} trait TwitterHandler extends Service[Request, Response] { private[this] val log = Logger[TwitterHandler] @@ -18,9 +17,6 @@ trait TwitterHandler extends Service[Request, Response] { content = Buf.Utf8(msg) ) - protected[handler] def getParams(uri: String): Map[String, Seq[String]] = - parse(uri)._2 - protected def log(req: Request, msg: String): Unit = { log.info("[%s %s] %s".format(req.method, req.uri, msg)) } diff --git a/server/src/main/scala/com/twitter/server/util/HttpUtils.scala b/server/src/main/scala/com/twitter/server/util/HttpUtils.scala index 5c52da2b..ed8cf7e5 100644 --- a/server/src/main/scala/com/twitter/server/util/HttpUtils.scala +++ b/server/src/main/scala/com/twitter/server/util/HttpUtils.scala @@ -5,8 +5,6 @@ import com.twitter.finagle.http.{MediaType, Request, Response, Status, Version, import com.twitter.io.Buf import com.twitter.util.{Future, Time, Closable} import io.netty.handler.codec.http.QueryStringDecoder -import scala.collection.JavaConverters._ -import scala.collection.{Map, Seq} private[server] object HttpUtils { @@ -94,10 +92,4 @@ private[server] object HttpUtils { content = Buf.Utf8(msg) ) - /** Parse uri into (path, params) */ - def parse(uri: String): (String, Map[String, Seq[String]]) = { - val qsd = new QueryStringDecoder(uri) - val params = qsd.parameters().asScala.mapValues(_.asScala.toSeq) - (qsd.path(), params) - } } diff --git a/server/src/test/scala/com/twitter/server/util/HttpUtilsTest.scala b/server/src/test/scala/com/twitter/server/util/HttpUtilsTest.scala index 3701aef7..e781f1c0 100644 --- a/server/src/test/scala/com/twitter/server/util/HttpUtilsTest.scala +++ b/server/src/test/scala/com/twitter/server/util/HttpUtilsTest.scala @@ -192,18 +192,4 @@ class HttpUtilsTest extends FunSuite { assert(res.contentString == "not found") } - test("Extract query values") { - val uri = "http://test.com/testing?foo=bar&baz=qux&hello=world" - val (_, params) = parse(uri) - assert(params("foo") == Seq("bar")) - assert(params.get("fun") == None) - } - - test("Extract multiple query values") { - val uri = "http://test.com/testing?foo=bar&baz=qux&foo=world" - val (_, params) = parse(uri) - assert(params("foo") == Seq("bar", "world")) - assert(params("baz") == Seq("qux")) - assert(params.get("fun") == None) - } } diff --git a/slf4j-jdk14/src/main/scala/com/twitter/server/handler/slf4j/jdk14/LoggingHandler.scala b/slf4j-jdk14/src/main/scala/com/twitter/server/handler/slf4j/jdk14/LoggingHandler.scala index 5b902d77..9a5c5fc1 100644 --- a/slf4j-jdk14/src/main/scala/com/twitter/server/handler/slf4j/jdk14/LoggingHandler.scala +++ b/slf4j-jdk14/src/main/scala/com/twitter/server/handler/slf4j/jdk14/LoggingHandler.scala @@ -100,10 +100,11 @@ private class LoggingHandler extends AdminHttpMuxHandler { ) def apply(request: Request): Future[Response] = { - val (_, params) = parse(request.uri) + val uri = Uri.fromRequest(request) + val params = uri.params val loggerName: Option[String] = parseLoggerName(params) - val loggerLevel: Option[String] = params.getOrElse("level", Seq.empty).headOption + val loggerLevel: Option[String] = params.get("level") val updateMsg = (loggerLevel, loggerName) match { case (Some(level), Some(name)) => @@ -137,10 +138,8 @@ private class LoggingHandler extends AdminHttpMuxHandler { } } - private[this] def parseLoggerName( - params: scala.collection.Map[String, scala.collection.Seq[String]] - ): Option[String] = { - params.getOrElse("logger", Seq.empty).headOption.map { + private[this] def parseLoggerName(params: ParamMap): Option[String] = { + params.get("logger").map { case "root" => "" case name => name } From ecef399ad98f5386cd75c031a8a2e73d2159a575 Mon Sep 17 00:00:00 2001 From: Ian Bennett Date: Wed, 13 Nov 2019 19:28:03 +0000 Subject: [PATCH 115/409] twitter-server: introduce ability to prevent the `AdminHttpServer` from starting Problem The `AdminHttpServer`contains mutable state that allows for a server to be started again after it is closed. Additionally, there are lifecycle concerns within Finatra that allow for disabling the `AdminHttpServer` from starting does so by always starting the `AdminHttpServer` and closing it post-warmup. Finatra also suffers from a flaky initialization order issue if the `disableAdminHttpServer` property is defined as a val. Solution We handle the case where an `AdminHttpServer` can be closed, then re-assigned/re-opened by ensuring that it can only be closed once and fail traffic after closing. Additionally, we thread the lifecycle concerns of disabling the `AdminHttpServer` through the `AdminHttpServer` and ensure that if the server is disabled, it is never started. We also ensure that `val` and `def` overrides of `disableAdminHttpServer` behave consistently. JIRA Issues: CSL-9025 Differential Revision: https://phabricator.twitter.biz/D397925 --- CHANGELOG.rst | 4 ++ .../com/twitter/server/AdminHttpServer.scala | 17 ++++++- .../twitter/server/AdminHttpServerTest.scala | 46 +++++++++++++++++++ 3 files changed, 66 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b4105665..13e3a5bf 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -16,6 +16,10 @@ Unreleased * The signature of the `getGraceParam` method of `ShutdownHandler` has been changed to take a `Request`. +* Add a `disableAdminHttpServer` property to the `AdminHttpServer` that can be used to + prevent the `AdminHttpServer` from starting by default on a `TwitterServer`. ``PHAB_ID=D397925`` + + 19.11.0 ------- diff --git a/server/src/main/scala/com/twitter/server/AdminHttpServer.scala b/server/src/main/scala/com/twitter/server/AdminHttpServer.scala index 87de8f16..0ee92dd3 100644 --- a/server/src/main/scala/com/twitter/server/AdminHttpServer.scala +++ b/server/src/main/scala/com/twitter/server/AdminHttpServer.scala @@ -108,6 +108,12 @@ trait AdminHttpServer { self: App with Stats => // We use slf4-api directly b/c we're in a trait and want the trait class to be the Logger name private[this] val log = LoggerFactory.getLogger(getClass) + /** + * If true, the Twitter-Server admin server will be disabled. + * Note: Disabling the admin server allows services to be deployed into environments where only a single port is allowed + */ + protected def disableAdminHttpServer: Boolean = false + def defaultAdminPort: Int = 9990 val adminPort: Flag[InetSocketAddress] = flag("admin.port", new InetSocketAddress(defaultAdminPort), "Admin http server port") @@ -285,7 +291,16 @@ trait AdminHttpServer { self: App with Stats => } premain { + // For consistency, we will add the routes regardless of whether the `adminHttpServer` gets + // started. This may not always be true and we may change this behavior in the future. addAdminRoutes(Admin.adminRoutes(statsReceiver, self)) - startServer() + + // we delay this check until we call the premain to ensure the `disableAdminHttpServer` value + // has the correct initialization order + if (disableAdminHttpServer) { + log.info("admin http is disabled and will not be started.") + } else { + startServer() + } } } diff --git a/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala b/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala index d2cdf3ff..236d1546 100644 --- a/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala +++ b/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala @@ -103,6 +103,52 @@ class AdminHttpServerTest extends FunSuite with Eventually with IntegrationPatie assert(MockClosableHandler.closed) } + test("a closed server cannot be restarted") { + val server = new TestTwitterServer { + override def main(): Unit = { + Await.result(adminHttpServer.close(5.seconds), 7.seconds) + intercept[Exception] { + checkServer(adminHttpServer) + } + + val adminServerBoundPort = + adminHttpServer.boundAddress.asInstanceOf[InetSocketAddress].getPort + assert(adminServerBoundPort == adminBoundAddress.getPort) + val client = Http.client.newService(s"localhost:$adminServerBoundPort") + intercept[Exception] { // the admin is already closed, this should error + Await.result(client(Request(Method.Post, "/quitquitquit")), 1.second) + } + + Await.result(close(5.seconds), 7.seconds) + } + } + server.main(args = Array.empty[String]) + assert(MockClosableHandler.closed) + } + + test("a disabled server cannot be started") { + val server = new TestTwitterServer { + override protected def disableAdminHttpServer: Boolean = true + override def main(): Unit = { + intercept[Exception] { + checkServer(adminHttpServer) + } + + val adminServerBoundPort = + adminHttpServer.boundAddress.asInstanceOf[InetSocketAddress].getPort + assert(adminServerBoundPort == adminBoundAddress.getPort) + val client = Http.client.newService(s"localhost:$adminServerBoundPort") + intercept[Exception] { // the admin is already closed, this should error + Await.result(client(Request(Method.Post, "/quitquitquit")), 1.second) + } + + Await.result(close(5.seconds), 7.seconds) + } + } + server.main(args = Array.empty[String]) + assert(MockClosableHandler.closed) + } + test("GET does not close server") { val server = new TestTwitterServer { override def main(): Unit = { From 69fb2186705b4d9e48aae8e520e8d66c2677230a Mon Sep 17 00:00:00 2001 From: Hamdi Allam Date: Thu, 14 Nov 2019 03:52:05 +0000 Subject: [PATCH 116/409] t-s-i: duplicate imports & formatting Problem duplicate import in the filters package Solution / Result: removed and formatted imports Differential Revision: https://phabricator.twitter.biz/D400207 --- .../com/twitter/server/filters/AdminThreadPoolFilter.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/filters/AdminThreadPoolFilter.scala b/server/src/main/scala/com/twitter/server/filters/AdminThreadPoolFilter.scala index 626dc0c4..e46cee2c 100644 --- a/server/src/main/scala/com/twitter/server/filters/AdminThreadPoolFilter.scala +++ b/server/src/main/scala/com/twitter/server/filters/AdminThreadPoolFilter.scala @@ -1,10 +1,9 @@ package com.twitter.server.filters import com.twitter.concurrent.NamedPoolThreadFactory -import com.twitter.finagle.{Service} +import com.twitter.finagle.Service import com.twitter.finagle.filter.OffloadFilter import com.twitter.finagle.http.{Request, Response} -import com.twitter.finagle.filter.OffloadFilter import com.twitter.server.AdminHttpServer.Route import com.twitter.util.{ExecutorServiceFuturePool, FuturePool} import java.util.concurrent.Executors From c9defbf60202595e2840e9869fc94f6815ad52b9 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Thu, 14 Nov 2019 18:27:11 +0000 Subject: [PATCH 117/409] util, finagle, twitter-server: Remove JavaSingleton Trait Problem / Solution The `JavaSingleton` trait in util is lightly used and doesn't provide functionality in the way that it's intended to work. Let's remove its single uses within Finagle and Twitter Server and encourage folks to take a different approach for better Java compatibility. JIRA Issues: CSL-9080 Differential Revision: https://phabricator.twitter.biz/D399947 --- CHANGELOG.rst | 2 ++ .../scala/com/twitter/server/handler/ResourceHandler.scala | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 13e3a5bf..c36bb462 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -19,6 +19,8 @@ Unreleased * Add a `disableAdminHttpServer` property to the `AdminHttpServer` that can be used to prevent the `AdminHttpServer` from starting by default on a `TwitterServer`. ``PHAB_ID=D397925`` +* The `ResourceHandler` companion object is no longer a `JavaSingleton`. + ``PHAB_ID=D399947`` 19.11.0 ------- diff --git a/server/src/main/scala/com/twitter/server/handler/ResourceHandler.scala b/server/src/main/scala/com/twitter/server/handler/ResourceHandler.scala index c4a3a2b9..343842e3 100644 --- a/server/src/main/scala/com/twitter/server/handler/ResourceHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/ResourceHandler.scala @@ -4,7 +4,7 @@ import com.twitter.finagle.Service import com.twitter.finagle.http.{Request, Response, Status, Uri} import com.twitter.io.Buf import com.twitter.server.util.HttpUtils.{new404, newResponse} -import com.twitter.util.{Future, FuturePool, JavaSingleton} +import com.twitter.util.{Future, FuturePool} import java.io.{File, FileInputStream, InputStream} import java.nio.charset.{Charset, StandardCharsets} import scala.io.Source @@ -63,7 +63,7 @@ class ResourceHandler( } } -object ResourceHandler extends JavaSingleton { +object ResourceHandler { /** * Constructs a ResourceHandler which tries to read resources from disk on every request, From 16ae22676a5ce8b0c3e1dd30394c0e37b481346c Mon Sep 17 00:00:00 2001 From: Hamdi Allam Date: Thu, 14 Nov 2019 20:59:32 +0000 Subject: [PATCH 118/409] twitter-server: flakey AdminThreadPoolFilter test Problem: AdminThreadPoolFilter checks `numCompletedTasks` on the admin thread pool at the end of the test. `numCompletedTasks` is specified to be an approximation in the JavaDoc. Solution / Result: Remove this specific assert. `AdminThreadPoolFilter.Pool` no longer has to be packaged scoped either. The test already ensures that the filter shifts work to another thread. JIRA Issues: CSL-9087 Differential Revision: https://phabricator.twitter.biz/D400449 --- .../com/twitter/server/filters/AdminThreadPoolFilter.scala | 3 +-- .../com/twitter/server/filters/AdminThreadPoolFilterTest.scala | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/filters/AdminThreadPoolFilter.scala b/server/src/main/scala/com/twitter/server/filters/AdminThreadPoolFilter.scala index e46cee2c..4062b3a0 100644 --- a/server/src/main/scala/com/twitter/server/filters/AdminThreadPoolFilter.scala +++ b/server/src/main/scala/com/twitter/server/filters/AdminThreadPoolFilter.scala @@ -9,8 +9,7 @@ import com.twitter.util.{ExecutorServiceFuturePool, FuturePool} import java.util.concurrent.Executors object AdminThreadPoolFilter { - // scoped for testing - private[filters] lazy val Pool: FuturePool = + private lazy val Pool: FuturePool = new ExecutorServiceFuturePool( Executors .newCachedThreadPool(new NamedPoolThreadFactory("AdminFuturePool", makeDaemons = true)) diff --git a/server/src/test/scala/com/twitter/server/filters/AdminThreadPoolFilterTest.scala b/server/src/test/scala/com/twitter/server/filters/AdminThreadPoolFilterTest.scala index 508c58a9..8649936c 100644 --- a/server/src/test/scala/com/twitter/server/filters/AdminThreadPoolFilterTest.scala +++ b/server/src/test/scala/com/twitter/server/filters/AdminThreadPoolFilterTest.scala @@ -25,6 +25,5 @@ class AdminThreadPoolFilterTest extends FunSuite { assert(workingThreadId != -1) assert(workingThreadId != caller.getId) - assert(AdminThreadPoolFilter.Pool.numCompletedTasks == 1) } } From 46dc8611fd2c601d8cb6f2cc1d1bbec2b000e347 Mon Sep 17 00:00:00 2001 From: Eitan Adler Date: Thu, 21 Nov 2019 05:16:02 +0000 Subject: [PATCH 119/409] [logging] lift log4j-over-slf4j from libraries into binaries Problem slfj is a logging facade with several implementations - which may be loaded at runtime. It is generally incorrect to include an implementation in a library. Instead, include just the API in library code and allow applications to depend on a specific implementation in the binary Solution This diff lifts one of the implementations from libraries into binaries JIRA Issues: AITOOLS-5215 Differential Revision: https://phabricator.twitter.biz/D402308 --- CONFIG.ini | 2 +- CONTRIBUTING.md | 2 +- README.md | 2 +- build.sbt | 2 +- doc/src/sphinx/Features.rst | 2 +- doc/src/sphinx/_templates/sidebarintro.html | 2 +- doc/src/sphinx/_themes/flask/layout.html | 2 +- .../sphinx/_themes/flask_small/layout.html | 2 +- doc/src/sphinx/links.txt | 2 +- logback-classic/BUILD | 6 +-- logback-classic/README.md | 2 + logback-classic/src/main/resources/BUILD | 5 +- logback-classic/src/main/scala/BUILD | 51 ++++++++++--------- logback-classic/src/test/scala/BUILD | 24 ++++----- server/src/main/resources/BUILD | 5 +- .../fonts/glyphicons-halflings-regular.svg | 2 +- server/src/main/scala/BUILD | 2 +- .../main/scala/com/twitter/server/Hook.scala | 2 +- .../main/scala/com/twitter/server/Hooks.scala | 2 +- .../scala/com/twitter/server/NewHook.scala | 2 +- .../twitter/server/lint/LoggingRules.scala | 2 +- .../twitter/server/util/MetricSource.scala | 2 +- .../com/twitter/server/view/ThreadsView.scala | 2 +- server/src/test/resources/BUILD | 5 +- .../scala/com/twitter/server/HookTest.scala | 2 +- .../twitter/server/TwitterServerTest.scala | 2 +- .../handler/ClientRegistryHandlerTest.scala | 2 +- .../server/util/MetricSourceTest.scala | 2 +- slf4j-jdk14/BUILD | 6 +-- slf4j-jdk14/README.md | 4 +- slf4j-jdk14/src/main/resources/BUILD | 7 ++- slf4j-jdk14/src/main/scala/BUILD | 49 +++++++++--------- slf4j-jdk14/src/test/scala/BUILD | 26 +++++----- slf4j-log4j12/BUILD | 6 +-- slf4j-log4j12/README.md | 2 +- slf4j-log4j12/src/main/resources/BUILD | 5 +- slf4j-log4j12/src/main/scala/BUILD | 2 +- slf4j-log4j12/src/test/scala/BUILD | 24 ++++----- 38 files changed, 145 insertions(+), 126 deletions(-) diff --git a/CONFIG.ini b/CONFIG.ini index f646bbc8..44f14ab4 100644 --- a/CONFIG.ini +++ b/CONFIG.ini @@ -5,4 +5,4 @@ project: CSL [docbird] project = Twitter Server project_type = library -docs_dir = ./doc/src/sphinx \ No newline at end of file +docs_dir = ./doc/src/sphinx diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 534c7c78..500af2c5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -241,4 +241,4 @@ requests that improve the existing Scaladocs! [travis-ci]: https://travis-ci.org/twitter/twitter-server [changes]: https://github.com/twitter/twitter-server/blob/develop/CHANGELOG.rst [scalacheck]: https://www.scalacheck.org/ -[gendrivenprop]: https://www.scalatest.org/user_guide/generator_driven_property_checks \ No newline at end of file +[gendrivenprop]: https://www.scalatest.org/user_guide/generator_driven_property_checks diff --git a/README.md b/README.md index dfacefc9..5baf251e 100644 --- a/README.md +++ b/README.md @@ -44,4 +44,4 @@ The `master` branch of this repository contains the latest stable release of TwitterServer, and weekly snapshots are published to the `develop` branch. In general pull requests should be submitted against `develop`. See [CONTRIBUTING.md](https://github.com/twitter/twitter-server/blob/master/CONTRIBUTING.md) -for more details about how to contribute. \ No newline at end of file +for more details about how to contribute. diff --git a/build.sbt b/build.sbt index 2554e887..ce16eb5a 100644 --- a/build.sbt +++ b/build.sbt @@ -284,4 +284,4 @@ lazy val twitterServerDoc = (project in file("doc")) .dependsOn(twitterServer) /* Test Configuration for running tests on doc sources */ -lazy val DocTest = config("testdoc") extend Test \ No newline at end of file +lazy val DocTest = config("testdoc") extend Test diff --git a/doc/src/sphinx/Features.rst b/doc/src/sphinx/Features.rst index f1d22e6c..0cf6a3d7 100644 --- a/doc/src/sphinx/Features.rst +++ b/doc/src/sphinx/Features.rst @@ -517,4 +517,4 @@ trait is built. .. _twitter-server/logback-classic : https://github.com/twitter/twitter-server/tree/develop/logback-classic .. |com.twitter.server.logging.Logging| replace:: ``com.twitter.server.logging.Logging`` trait -.. _com.twitter.server.logging.Logging: https://github.com/twitter/twitter-server/blob/develop/slf4j-jdk14/src/main/scala/com/twitter/server/logging/Logging.scala \ No newline at end of file +.. _com.twitter.server.logging.Logging: https://github.com/twitter/twitter-server/blob/develop/slf4j-jdk14/src/main/scala/com/twitter/server/logging/Logging.scala diff --git a/doc/src/sphinx/_templates/sidebarintro.html b/doc/src/sphinx/_templates/sidebarintro.html index 2f672326..a56f6e5c 100644 --- a/doc/src/sphinx/_templates/sidebarintro.html +++ b/doc/src/sphinx/_templates/sidebarintro.html @@ -7,4 +7,4 @@

    Useful Links

    \ No newline at end of file + diff --git a/doc/src/sphinx/_themes/flask/layout.html b/doc/src/sphinx/_themes/flask/layout.html index 9f018809..3ea8602e 100644 --- a/doc/src/sphinx/_themes/flask/layout.html +++ b/doc/src/sphinx/_themes/flask/layout.html @@ -22,4 +22,4 @@ {% if pagename == 'index' %} {% endif %} -{%- endblock %} \ No newline at end of file +{%- endblock %} diff --git a/doc/src/sphinx/_themes/flask_small/layout.html b/doc/src/sphinx/_themes/flask_small/layout.html index b45c8a58..83e52139 100644 --- a/doc/src/sphinx/_themes/flask_small/layout.html +++ b/doc/src/sphinx/_themes/flask_small/layout.html @@ -19,4 +19,4 @@ {% endif %} {% endblock %} {% block sidebar1 %}{% endblock %} -{% block sidebar2 %}{% endblock %} \ No newline at end of file +{% block sidebar2 %}{% endblock %} diff --git a/doc/src/sphinx/links.txt b/doc/src/sphinx/links.txt index 3a341e48..21a64376 100644 --- a/doc/src/sphinx/links.txt +++ b/doc/src/sphinx/links.txt @@ -1 +1 @@ -.. _Sphinx: https://www.sphinx-doc.org/en/master/ \ No newline at end of file +.. _Sphinx: https://www.sphinx-doc.org/en/master/ diff --git a/logback-classic/BUILD b/logback-classic/BUILD index 5aeeaaeb..7cc8414b 100644 --- a/logback-classic/BUILD +++ b/logback-classic/BUILD @@ -1,5 +1,5 @@ target( - dependencies=[ - 'twitter-server/logback-classic/src/main/scala' - ] + dependencies = [ + "twitter-server/logback-classic/src/main/scala", + ], ) diff --git a/logback-classic/README.md b/logback-classic/README.md index c9c1e3fb..4638a03b 100644 --- a/logback-classic/README.md +++ b/logback-classic/README.md @@ -17,3 +17,5 @@ Depending on this library will install a logging handler on the [HTTP admin inte which allows users to dynamically change `ch.qos.logback.classic.Logger` log levels. For more information see the `TwitterServer` user guide section on [Logging](https://twitter.github.io/twitter-server/Features.html#logging) + + diff --git a/logback-classic/src/main/resources/BUILD b/logback-classic/src/main/resources/BUILD index 7267daa1..e0afa86a 100644 --- a/logback-classic/src/main/resources/BUILD +++ b/logback-classic/src/main/resources/BUILD @@ -1,3 +1,6 @@ resources( - sources=rglobs('*', exclude=[rglobs('BUILD*', '*.pyc')]) + sources = rglobs( + "*", + exclude = [rglobs("BUILD*", "*.pyc")], + ), ) diff --git a/logback-classic/src/main/scala/BUILD b/logback-classic/src/main/scala/BUILD index 1e664307..eb95a09a 100644 --- a/logback-classic/src/main/scala/BUILD +++ b/logback-classic/src/main/scala/BUILD @@ -1,27 +1,28 @@ scala_library( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, - provides = scala_artifact( - org = "com.twitter", - name = "twitter-server-logback-classic", - repo = artifactory, - ), - scope = "forced", # Exposes a service-loaded class. - dependencies = [ - "3rdparty/jvm/ch/qos/logback:logback-classic", - "3rdparty/jvm/ch/qos/logback:logback-core", - "3rdparty/jvm/org/slf4j:jcl-over-slf4j", - "3rdparty/jvm/org/slf4j:jul-to-slf4j", - "3rdparty/jvm/org/slf4j:log4j-over-slf4j", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finagle/finagle-base-http/src/main/scala", - "finagle/finagle-http/src/main/scala", - "twitter-server/logback-classic/src/main/resources", - "twitter-server/server/src/main/scala", - "util/util-slf4j-api/src/main/scala", - ], - exports = [ - "3rdparty/jvm/ch/qos/logback:logback-classic", - "3rdparty/jvm/org/slf4j:slf4j-api", - ], + sources = rglobs("*.scala"), + compiler_option_sets = {"fatal_warnings"}, + provides = scala_artifact( + org = "com.twitter", + name = "twitter-server-logback-classic", + repo = artifactory, + ), + scope = "forced", # Exposes a service-loaded class. + tags = ["logging_impl_check_whitelisted_target"], + dependencies = [ + "3rdparty/jvm/ch/qos/logback:logback-classic", + "3rdparty/jvm/ch/qos/logback:logback-core", + "3rdparty/jvm/org/slf4j:jcl-over-slf4j", + "3rdparty/jvm/org/slf4j:jul-to-slf4j", + "3rdparty/jvm/org/slf4j:log4j-over-slf4j", + "3rdparty/jvm/org/slf4j:slf4j-api", + "finagle/finagle-base-http/src/main/scala", + "finagle/finagle-http/src/main/scala", + "twitter-server/logback-classic/src/main/resources", + "twitter-server/server/src/main/scala", + "util/util-slf4j-api/src/main/scala", + ], + exports = [ + "3rdparty/jvm/ch/qos/logback:logback-classic", + "3rdparty/jvm/org/slf4j:slf4j-api", + ], ) diff --git a/logback-classic/src/test/scala/BUILD b/logback-classic/src/test/scala/BUILD index 4ec8347c..09a13a20 100644 --- a/logback-classic/src/test/scala/BUILD +++ b/logback-classic/src/test/scala/BUILD @@ -1,14 +1,14 @@ junit_tests( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, - dependencies = [ - "3rdparty/jvm/ch/qos/logback:logback-classic", - "3rdparty/jvm/junit", - "3rdparty/jvm/org/scalatest", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finagle/finagle-core/src/main/scala", - "twitter-server/logback-classic/src/main/scala", - "twitter-server/server/src/main/scala", - "util/util-slf4j-api/src/main/scala", - ], + sources = rglobs("*.scala"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/ch/qos/logback:logback-classic", + "3rdparty/jvm/junit", + "3rdparty/jvm/org/scalatest", + "3rdparty/jvm/org/slf4j:slf4j-api", + "finagle/finagle-core/src/main/scala", + "twitter-server/logback-classic/src/main/scala", + "twitter-server/server/src/main/scala", + "util/util-slf4j-api/src/main/scala", + ], ) diff --git a/server/src/main/resources/BUILD b/server/src/main/resources/BUILD index 7267daa1..e0afa86a 100644 --- a/server/src/main/resources/BUILD +++ b/server/src/main/resources/BUILD @@ -1,3 +1,6 @@ resources( - sources=rglobs('*', exclude=[rglobs('BUILD*', '*.pyc')]) + sources = rglobs( + "*", + exclude = [rglobs("BUILD*", "*.pyc")], + ), ) diff --git a/server/src/main/resources/twitter-server/fonts/glyphicons-halflings-regular.svg b/server/src/main/resources/twitter-server/fonts/glyphicons-halflings-regular.svg index 5b849823..326d348d 100644 --- a/server/src/main/resources/twitter-server/fonts/glyphicons-halflings-regular.svg +++ b/server/src/main/resources/twitter-server/fonts/glyphicons-halflings-regular.svg @@ -226,4 +226,4 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/server/src/main/scala/BUILD b/server/src/main/scala/BUILD index 48cb57c5..d6135845 100644 --- a/server/src/main/scala/BUILD +++ b/server/src/main/scala/BUILD @@ -44,4 +44,4 @@ scala_library( "util/util-lint/src/main/scala", "util/util-slf4j-api/src/main/scala", ], -) \ No newline at end of file +) diff --git a/server/src/main/scala/com/twitter/server/Hook.scala b/server/src/main/scala/com/twitter/server/Hook.scala index cf32a667..f54d8670 100644 --- a/server/src/main/scala/com/twitter/server/Hook.scala +++ b/server/src/main/scala/com/twitter/server/Hook.scala @@ -15,4 +15,4 @@ abstract class Hook { * run in parallel and are executed after all `App#postmain` functions complete. */ def onExit(): Unit = () -} \ No newline at end of file +} diff --git a/server/src/main/scala/com/twitter/server/Hooks.scala b/server/src/main/scala/com/twitter/server/Hooks.scala index facd709a..8bb2f6da 100644 --- a/server/src/main/scala/com/twitter/server/Hooks.scala +++ b/server/src/main/scala/com/twitter/server/Hooks.scala @@ -18,4 +18,4 @@ trait Hooks { self: App => for (hook <- hooks) onExit { hook.onExit() } -} \ No newline at end of file +} diff --git a/server/src/main/scala/com/twitter/server/NewHook.scala b/server/src/main/scala/com/twitter/server/NewHook.scala index 887e6933..f0c6dc63 100644 --- a/server/src/main/scala/com/twitter/server/NewHook.scala +++ b/server/src/main/scala/com/twitter/server/NewHook.scala @@ -25,4 +25,4 @@ import com.twitter.app.App * @see [[com.twitter.server.Hook]] * @see [[com.twitter.finagle.util.LoadService]] */ -trait NewHook extends (App => Hook) \ No newline at end of file +trait NewHook extends (App => Hook) diff --git a/server/src/main/scala/com/twitter/server/lint/LoggingRules.scala b/server/src/main/scala/com/twitter/server/lint/LoggingRules.scala index 6f9a7610..7c3dd9bd 100644 --- a/server/src/main/scala/com/twitter/server/lint/LoggingRules.scala +++ b/server/src/main/scala/com/twitter/server/lint/LoggingRules.scala @@ -50,4 +50,4 @@ object LoggingRules { ) { Seq(Issue("No logging handler implementation configured.")) } -} \ No newline at end of file +} diff --git a/server/src/main/scala/com/twitter/server/util/MetricSource.scala b/server/src/main/scala/com/twitter/server/util/MetricSource.scala index 95309266..299a2e99 100644 --- a/server/src/main/scala/com/twitter/server/util/MetricSource.scala +++ b/server/src/main/scala/com/twitter/server/util/MetricSource.scala @@ -77,4 +77,4 @@ private[server] class MetricSource( case (name, statEntry) => MetricSource.MetricTypeInfo(name, statEntry.metricType) } } -} \ No newline at end of file +} diff --git a/server/src/main/scala/com/twitter/server/view/ThreadsView.scala b/server/src/main/scala/com/twitter/server/view/ThreadsView.scala index 985d9dda..bf256be8 100644 --- a/server/src/main/scala/com/twitter/server/view/ThreadsView.scala +++ b/server/src/main/scala/com/twitter/server/view/ThreadsView.scala @@ -152,4 +152,4 @@ private[server] class ThreadsView(all: Seq[ThreadInfo], deadlockedIds: Seq[Long] ScriptFooter } -} \ No newline at end of file +} diff --git a/server/src/test/resources/BUILD b/server/src/test/resources/BUILD index 7267daa1..e0afa86a 100644 --- a/server/src/test/resources/BUILD +++ b/server/src/test/resources/BUILD @@ -1,3 +1,6 @@ resources( - sources=rglobs('*', exclude=[rglobs('BUILD*', '*.pyc')]) + sources = rglobs( + "*", + exclude = [rglobs("BUILD*", "*.pyc")], + ), ) diff --git a/server/src/test/scala/com/twitter/server/HookTest.scala b/server/src/test/scala/com/twitter/server/HookTest.scala index dd3093cb..3bfd2d2b 100644 --- a/server/src/test/scala/com/twitter/server/HookTest.scala +++ b/server/src/test/scala/com/twitter/server/HookTest.scala @@ -71,4 +71,4 @@ class HookTest extends FunSuite { Seq('Init, 'PreMain, 'Main, 'PostMain, 'Exit) ) } -} \ No newline at end of file +} diff --git a/server/src/test/scala/com/twitter/server/TwitterServerTest.scala b/server/src/test/scala/com/twitter/server/TwitterServerTest.scala index d06dfaa6..337c2c28 100644 --- a/server/src/test/scala/com/twitter/server/TwitterServerTest.scala +++ b/server/src/test/scala/com/twitter/server/TwitterServerTest.scala @@ -67,4 +67,4 @@ class TwitterServerTest extends FunSuite { twitterServer.main(args = Array.empty[String]) assert(twitterServer.bootstrapSeq == Seq('Init, 'PreMain, 'Main, 'Exit, 'PostMain)) } -} \ No newline at end of file +} diff --git a/server/src/test/scala/com/twitter/server/handler/ClientRegistryHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/ClientRegistryHandlerTest.scala index c76bbc44..965aa28b 100644 --- a/server/src/test/scala/com/twitter/server/handler/ClientRegistryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/ClientRegistryHandlerTest.scala @@ -81,4 +81,4 @@ class ClientRegistryHandlerTest extends FunSuite { assert(html.contains("98.00%")) } } -} \ No newline at end of file +} diff --git a/server/src/test/scala/com/twitter/server/util/MetricSourceTest.scala b/server/src/test/scala/com/twitter/server/util/MetricSourceTest.scala index 731af011..6c185923 100644 --- a/server/src/test/scala/com/twitter/server/util/MetricSourceTest.scala +++ b/server/src/test/scala/com/twitter/server/util/MetricSourceTest.scala @@ -63,4 +63,4 @@ class MetricSourceTest extends FunSuite { assert(source.keySet == Set("clnt/foo/requests", "clnt/foo/success")) } } -} \ No newline at end of file +} diff --git a/slf4j-jdk14/BUILD b/slf4j-jdk14/BUILD index 117ef0cf..c8baf98e 100644 --- a/slf4j-jdk14/BUILD +++ b/slf4j-jdk14/BUILD @@ -1,5 +1,5 @@ target( - dependencies=[ - 'twitter-server/slf4j-jdk14/src/main/scala' - ] + dependencies = [ + "twitter-server/slf4j-jdk14/src/main/scala", + ], ) diff --git a/slf4j-jdk14/README.md b/slf4j-jdk14/README.md index 972f7638..672f0c67 100644 --- a/slf4j-jdk14/README.md +++ b/slf4j-jdk14/README.md @@ -29,7 +29,7 @@ logging. Admin HTTP Interface Logging Handler ------------------------------------ -Depending on this library will install a logging handler on the [HTTP admin interface](https://twitter.github.io/Admin.html#admin-interface) +Depending on this library will install a logging handler on the [HTTP admin interface](https://twitter.github.io/twitter-server/Admin.html#admin-interface) which allows users to dynamically change `com.twitter.logging.Logger` log levels. -For more information see the `TwitterServer` user guide section on [Logging](https://twitter.github.io/Features.html#logging) +For more information see the `TwitterServer` user guide section on [Logging](https://twitter.github.io/twitter-server/Features.html#logging) diff --git a/slf4j-jdk14/src/main/resources/BUILD b/slf4j-jdk14/src/main/resources/BUILD index 4e5c40ee..e0afa86a 100644 --- a/slf4j-jdk14/src/main/resources/BUILD +++ b/slf4j-jdk14/src/main/resources/BUILD @@ -1,3 +1,6 @@ resources( - sources=rglobs('*', exclude=[rglobs('BUILD*', '*.pyc')]) -) \ No newline at end of file + sources = rglobs( + "*", + exclude = [rglobs("BUILD*", "*.pyc")], + ), +) diff --git a/slf4j-jdk14/src/main/scala/BUILD b/slf4j-jdk14/src/main/scala/BUILD index c9dadbf8..ed275eff 100644 --- a/slf4j-jdk14/src/main/scala/BUILD +++ b/slf4j-jdk14/src/main/scala/BUILD @@ -1,26 +1,27 @@ scala_library( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, - provides = scala_artifact( - org = "com.twitter", - name = "twitter-server-slf4j-jdk14", - repo = artifactory, - ), - scope = "forced", # Exposes a service-loaded class. - dependencies = [ - "3rdparty/jvm/org/slf4j:jcl-over-slf4j", - "3rdparty/jvm/org/slf4j:log4j-over-slf4j", - "3rdparty/jvm/org/slf4j:slf4j-api", - "3rdparty/jvm/org/slf4j:slf4j-jdk14", - "finagle/finagle-base-http/src/main/scala", - "finagle/finagle-http/src/main/scala", - "twitter-server/server/src/main/scala", - "twitter-server/slf4j-jdk14/src/main/resources", - "util/util-logging/src/main/scala", - ], - exports = [ - "3rdparty/jvm/org/slf4j:slf4j-api", - "3rdparty/jvm/org/slf4j:slf4j-jdk14", - "util/util-logging/src/main/scala", - ], + sources = rglobs("*.scala"), + compiler_option_sets = {"fatal_warnings"}, + provides = scala_artifact( + org = "com.twitter", + name = "twitter-server-slf4j-jdk14", + repo = artifactory, + ), + scope = "forced", # Exposes a service-loaded class. + tags = ["logging_impl_check_whitelisted_target"], + dependencies = [ + "3rdparty/jvm/org/slf4j:jcl-over-slf4j", + "3rdparty/jvm/org/slf4j:log4j-over-slf4j", + "3rdparty/jvm/org/slf4j:slf4j-api", + "3rdparty/jvm/org/slf4j:slf4j-jdk14", + "finagle/finagle-base-http/src/main/scala", + "finagle/finagle-http/src/main/scala", + "twitter-server/server/src/main/scala", + "twitter-server/slf4j-jdk14/src/main/resources", + "util/util-logging/src/main/scala", + ], + exports = [ + "3rdparty/jvm/org/slf4j:slf4j-api", + "3rdparty/jvm/org/slf4j:slf4j-jdk14", + "util/util-logging/src/main/scala", + ], ) diff --git a/slf4j-jdk14/src/test/scala/BUILD b/slf4j-jdk14/src/test/scala/BUILD index 6c615854..b9866b6d 100644 --- a/slf4j-jdk14/src/test/scala/BUILD +++ b/slf4j-jdk14/src/test/scala/BUILD @@ -1,15 +1,15 @@ junit_tests( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, - dependencies = [ - "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-all", - "3rdparty/jvm/org/scalacheck", - "3rdparty/jvm/org/scalatest", - "finagle/finagle-base-http/src/main/scala", - "finagle/finagle-core/src/main/scala", - "finagle/finagle-http/src/main/scala", - "twitter-server/server/src/main/scala", - "twitter-server/slf4j-jdk14/src/main/scala", - ], + sources = rglobs("*.scala"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/junit", + "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/scalacheck", + "3rdparty/jvm/org/scalatest", + "finagle/finagle-base-http/src/main/scala", + "finagle/finagle-core/src/main/scala", + "finagle/finagle-http/src/main/scala", + "twitter-server/server/src/main/scala", + "twitter-server/slf4j-jdk14/src/main/scala", + ], ) diff --git a/slf4j-log4j12/BUILD b/slf4j-log4j12/BUILD index 80f7ea1a..bfa2f4d4 100644 --- a/slf4j-log4j12/BUILD +++ b/slf4j-log4j12/BUILD @@ -1,5 +1,5 @@ target( - dependencies=[ - 'twitter-server/slf4j-log4j12/src/main/scala' - ] + dependencies = [ + "twitter-server/slf4j-log4j12/src/main/scala", + ], ) diff --git a/slf4j-log4j12/README.md b/slf4j-log4j12/README.md index 80ef6efa..ad2819f4 100644 --- a/slf4j-log4j12/README.md +++ b/slf4j-log4j12/README.md @@ -16,4 +16,4 @@ Admin HTTP Interface Logging Handler Depending on this library will install a logging handler on the [HTTP admin interface](https://twitter.github.io/twitter-server/Admin.html#admin-interface) which allows users to dynamically change `org.apache.log4j.Logger` log levels. -For more information see the `TwitterServer` user guide section on [Logging](https://twitter.github.io/twitter-server/Features.html#logging) \ No newline at end of file +For more information see the `TwitterServer` user guide section on [Logging](https://twitter.github.io/twitter-server/Features.html#logging) diff --git a/slf4j-log4j12/src/main/resources/BUILD b/slf4j-log4j12/src/main/resources/BUILD index 7267daa1..e0afa86a 100644 --- a/slf4j-log4j12/src/main/resources/BUILD +++ b/slf4j-log4j12/src/main/resources/BUILD @@ -1,3 +1,6 @@ resources( - sources=rglobs('*', exclude=[rglobs('BUILD*', '*.pyc')]) + sources = rglobs( + "*", + exclude = [rglobs("BUILD*", "*.pyc")], + ), ) diff --git a/slf4j-log4j12/src/main/scala/BUILD b/slf4j-log4j12/src/main/scala/BUILD index d0f3acde..9e8843aa 100644 --- a/slf4j-log4j12/src/main/scala/BUILD +++ b/slf4j-log4j12/src/main/scala/BUILD @@ -25,4 +25,4 @@ scala_library( "3rdparty/jvm/org/slf4j:slf4j-api", "3rdparty/jvm/org/slf4j:slf4j-log4j12", ], -) \ No newline at end of file +) diff --git a/slf4j-log4j12/src/test/scala/BUILD b/slf4j-log4j12/src/test/scala/BUILD index d733a26c..8a0ab6ef 100644 --- a/slf4j-log4j12/src/test/scala/BUILD +++ b/slf4j-log4j12/src/test/scala/BUILD @@ -1,14 +1,14 @@ junit_tests( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, - dependencies = [ - "3rdparty/jvm/junit", - "3rdparty/jvm/log4j", - "3rdparty/jvm/org/scalatest", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finagle/finagle-core/src/main/scala", - "twitter-server/server/src/main/scala", - "twitter-server/slf4j-log4j12/src/main/scala", - "util/util-slf4j-api/src/main/scala", - ], + sources = rglobs("*.scala"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/junit", + "3rdparty/jvm/log4j", + "3rdparty/jvm/org/scalatest", + "3rdparty/jvm/org/slf4j:slf4j-api", + "finagle/finagle-core/src/main/scala", + "twitter-server/server/src/main/scala", + "twitter-server/slf4j-log4j12/src/main/scala", + "util/util-slf4j-api/src/main/scala", + ], ) From f288eed87d59fa0c975b96ac53bf754247e0e05b Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Sat, 23 Nov 2019 00:51:22 +0000 Subject: [PATCH 120/409] Add ibennett to some OWNERS Problem/Solution ibennett/ibennett Differential Revision: https://phabricator.twitter.biz/D405285 --- OWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/OWNERS b/OWNERS index dc165413..995bde0b 100644 --- a/OWNERS +++ b/OWNERS @@ -1,5 +1,6 @@ banderson ccoco +ibennett koliver mnakamura roanta From a3477a22679c9d5fce2080ebcad526ce3d50a824 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Mon, 25 Nov 2019 20:42:56 +0000 Subject: [PATCH 121/409] Jackson: Update to 2.10.1 Problem/Solution Update Jackson to version `2.10.1`. The `ScalaObjectMapper` has been promoted out of `experimental` thus it's package has changed from `com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper` to `com.fasterxml.jackson.module.scala.ScalaObjectMapper`. The JsonNode API has also been updated and in some cases requires a specified type. More details in the Jackson release notes which helps to explain some of the changes herein: https://github.com/FasterXML/jackson/wiki/Jackson-Release-2.10 JIRA Issues: CSL-9115 Differential Revision: https://phabricator.twitter.biz/D402722 --- CHANGELOG.rst | 2 ++ build.sbt | 2 +- .../scala/com/twitter/server/HistogramQueryHandlerTest.scala | 3 +-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c36bb462..b0852fa9 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -22,6 +22,8 @@ Unreleased * The `ResourceHandler` companion object is no longer a `JavaSingleton`. ``PHAB_ID=D399947`` +* Upgrade Jackson dependency from version `2.9.9` to version `2.10.1` ``PHAB_ID=D402722`` + 19.11.0 ------- diff --git a/build.sbt b/build.sbt index ce16eb5a..c9a57ca1 100644 --- a/build.sbt +++ b/build.sbt @@ -3,7 +3,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch val releaseVersion = "19.12.0-SNAPSHOT" -val jacksonVersion = "2.9.9" +val jacksonVersion = "2.10.1" val jacksonLibs = Seq( "com.fasterxml.jackson.core" % "jackson-core" % jacksonVersion, "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion, diff --git a/server/src/test/scala/com/twitter/server/HistogramQueryHandlerTest.scala b/server/src/test/scala/com/twitter/server/HistogramQueryHandlerTest.scala index 33278d11..bef6171f 100644 --- a/server/src/test/scala/com/twitter/server/HistogramQueryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/HistogramQueryHandlerTest.scala @@ -1,8 +1,7 @@ package com.twitter.server import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.module.scala.DefaultScalaModule -import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper +import com.fasterxml.jackson.module.scala.{DefaultScalaModule, ScalaObjectMapper} import com.twitter.conversions.DurationOps._ import com.twitter.finagle.http.{Request, Response} import com.twitter.finagle.stats.{BucketAndCount, InMemoryStatsReceiver} From b36cdcedbdcf55db408ddf8f2c83341a5e8f4e22 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Tue, 3 Dec 2019 20:19:58 +0000 Subject: [PATCH 122/409] Revert "Jackson: Update to 2.10.1" Problem / Solution This change reverts the Jackson update to 2.10.1 due to incompatibilities with other existing libraries. JIRA Issues: CSL-9165 Differential Revision: https://phabricator.twitter.biz/D408069 --- CHANGELOG.rst | 2 -- build.sbt | 2 +- .../scala/com/twitter/server/HistogramQueryHandlerTest.scala | 3 ++- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b0852fa9..c36bb462 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -22,8 +22,6 @@ Unreleased * The `ResourceHandler` companion object is no longer a `JavaSingleton`. ``PHAB_ID=D399947`` -* Upgrade Jackson dependency from version `2.9.9` to version `2.10.1` ``PHAB_ID=D402722`` - 19.11.0 ------- diff --git a/build.sbt b/build.sbt index c9a57ca1..ce16eb5a 100644 --- a/build.sbt +++ b/build.sbt @@ -3,7 +3,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch val releaseVersion = "19.12.0-SNAPSHOT" -val jacksonVersion = "2.10.1" +val jacksonVersion = "2.9.9" val jacksonLibs = Seq( "com.fasterxml.jackson.core" % "jackson-core" % jacksonVersion, "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion, diff --git a/server/src/test/scala/com/twitter/server/HistogramQueryHandlerTest.scala b/server/src/test/scala/com/twitter/server/HistogramQueryHandlerTest.scala index bef6171f..33278d11 100644 --- a/server/src/test/scala/com/twitter/server/HistogramQueryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/HistogramQueryHandlerTest.scala @@ -1,7 +1,8 @@ package com.twitter.server import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.module.scala.{DefaultScalaModule, ScalaObjectMapper} +import com.fasterxml.jackson.module.scala.DefaultScalaModule +import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper import com.twitter.conversions.DurationOps._ import com.twitter.finagle.http.{Request, Response} import com.twitter.finagle.stats.{BucketAndCount, InMemoryStatsReceiver} From d9b1fc044a2f59abea6a1c6924c1ad25d834b083 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Tue, 3 Dec 2019 23:07:46 +0000 Subject: [PATCH 123/409] twitter-server: Update dependency versions Problem/Solution We have been using and building the framework against these versions internally for some time but the SBT build.sbt was not updated to reflect the version changes. Update the build.sbt to the correct versions. JIRA Issues: CSL-9121 Differential Revision: https://phabricator.twitter.biz/D408331 --- CHANGELOG.rst | 2 ++ CONTRIBUTING.md | 2 +- build.sbt | 4 ++-- .../test/scala/com/twitter/server/util/HtmlUtilsTest.scala | 4 ++-- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c36bb462..1d2508a8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -22,6 +22,8 @@ Unreleased * The `ResourceHandler` companion object is no longer a `JavaSingleton`. ``PHAB_ID=D399947`` +* Update ScalaTest to 3.0.8, and ScalaCheck to 1.14.0. ``PHAB_ID=D408331`` + 19.11.0 ------- diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 500af2c5..5c3b0802 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -83,7 +83,7 @@ locally). When appropriate, use [ScalaCheck][scalacheck] to write property-based tests for your code. This will often produce more thorough and effective inputs for your tests. We use ScalaTest's -[GeneratorDrivenPropertyChecks][gendrivenprop] as the entry point for +[ScalaCheckDrivenPropertyChecks][gendrivenprop] as the entry point for writing these tests. ## Compatibility diff --git a/build.sbt b/build.sbt index ce16eb5a..c3622403 100644 --- a/build.sbt +++ b/build.sbt @@ -84,8 +84,8 @@ lazy val sharedSettings = Seq( crossScalaVersions := Seq("2.11.12", "2.12.8"), fork in Test := true, // We have to fork to get the JavaOptions libraryDependencies ++= Seq( - "org.scalacheck" %% "scalacheck" % "1.13.4" % "test", - "org.scalatest" %% "scalatest" % "3.0.0" % "test", + "org.scalacheck" %% "scalacheck" % "1.14.0" % "test", + "org.scalatest" %% "scalatest" % "3.0.8" % "test", // See https://www.scala-sbt.org/0.13/docs/Testing.html#JUnit "com.novocode" % "junit-interface" % "0.11" % "test", "org.mockito" % "mockito-all" % "1.9.5" % "test" diff --git a/server/src/test/scala/com/twitter/server/util/HtmlUtilsTest.scala b/server/src/test/scala/com/twitter/server/util/HtmlUtilsTest.scala index 6aed69de..7bafc6bf 100644 --- a/server/src/test/scala/com/twitter/server/util/HtmlUtilsTest.scala +++ b/server/src/test/scala/com/twitter/server/util/HtmlUtilsTest.scala @@ -2,9 +2,9 @@ package com.twitter.server.util import org.scalacheck.Gen import org.scalatest.FunSuite -import org.scalatest.prop.GeneratorDrivenPropertyChecks +import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks -class HtmlUtilsTest extends FunSuite with GeneratorDrivenPropertyChecks { +class HtmlUtilsTest extends FunSuite with ScalaCheckDrivenPropertyChecks { private val escapedChars = Seq('>', '<', '&', '"', '\'') From acf7e01034902769eb128103a613155bcfc943c3 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Mon, 9 Dec 2019 23:55:31 +0000 Subject: [PATCH 124/409] jackson: Update to 2.9.10 (and jackson-databind to 2.9.10.1) Problem / Solution We want to upgrade to the latest version of jackson that we can, yet at this time upgrading to jackson 2.10 is problematic. Let's upgrade instead to the latest 2.9 version - 2.9.10.1. JIRA Issues: CSL-9183 Differential Revision: https://phabricator.twitter.biz/D410846 --- CHANGELOG.rst | 2 ++ build.sbt | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1d2508a8..8b6d131c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,8 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +* Upgrade to jackson 2.9.10 and jackson-databind 2.9.10.1 ``PHAB_ID=D410846`` + * Multiple changes have happened around query parameter retrieval in order to remove duplicated functionality from Twitter Server that exists in Finagle. Users are encouraged to use finagle-http's `Uri` class within their diff --git a/build.sbt b/build.sbt index c3622403..dbc81752 100644 --- a/build.sbt +++ b/build.sbt @@ -4,9 +4,10 @@ import scoverage.ScoverageKeys val releaseVersion = "19.12.0-SNAPSHOT" val jacksonVersion = "2.9.9" +val jacksonDatabindVersion = "2.9.10.1" val jacksonLibs = Seq( "com.fasterxml.jackson.core" % "jackson-core" % jacksonVersion, - "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion, + "com.fasterxml.jackson.core" % "jackson-databind" % jacksonDatabindVersion, "com.fasterxml.jackson.module" %% "jackson-module-scala" % jacksonVersion exclude("com.google.guava", "guava") ) val opencensusVersion = "0.19.1" From 1613e30624a716677097ef0a90290be59465a6e8 Mon Sep 17 00:00:00 2001 From: David Rusek Date: Thu, 12 Dec 2019 23:45:53 +0000 Subject: [PATCH 125/409] Twitter-oss: Prepare OSS libraries for release 19.12.0 Problem We want to release the next versions of our Twitter OSS libraries 19.12.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D413458 --- CHANGELOG.rst | 3 +++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8b6d131c..e8c09ba9 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +19.12.0 +------- + * Upgrade to jackson 2.9.10 and jackson-databind 2.9.10.1 ``PHAB_ID=D410846`` * Multiple changes have happened around query parameter retrieval in order diff --git a/README.md b/README.md index 5baf251e..7ff7d5c9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index dbc81752..dd7863a7 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.12.0-SNAPSHOT" +val releaseVersion = "19.12.0" val jacksonVersion = "2.9.9" val jacksonDatabindVersion = "2.9.10.1" From 9ac92e14172219d6380fcaff0d8bb8c171ac28b9 Mon Sep 17 00:00:00 2001 From: Eitan Adler Date: Tue, 17 Dec 2019 23:34:14 +0000 Subject: [PATCH 126/409] [logging] bump slf4j to 1.7.30 Problem slf4j is not at the latest version Solution Update slf4j to the latest version Result slf4j is at the latest version Differential Revision: https://phabricator.twitter.biz/D415019 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index dd7863a7..879b04a2 100644 --- a/build.sbt +++ b/build.sbt @@ -11,7 +11,7 @@ val jacksonLibs = Seq( "com.fasterxml.jackson.module" %% "jackson-module-scala" % jacksonVersion exclude("com.google.guava", "guava") ) val opencensusVersion = "0.19.1" -val slf4jVersion = "1.7.21" +val slf4jVersion = "1.7.30" def util(which: String) = "com.twitter" %% ("util-"+which) % releaseVersion def finagle(which: String) = "com.twitter" %% ("finagle-"+which) % releaseVersion From 02433e1ee39b7bbbdf3300d23fc1b914c9c2afe4 Mon Sep 17 00:00:00 2001 From: Eitan Adler Date: Thu, 19 Dec 2019 01:08:23 +0000 Subject: [PATCH 127/409] [logging] update logback to 1.2.3 Problem logback is out of date Solution logback should be update to date Result logback is up to date Differential Revision: https://phabricator.twitter.biz/D415888 --- CHANGELOG.rst | 6 ++++++ build.sbt | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e8c09ba9..e9e9b5cf 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,12 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Changed +~~~~~~~ + +* Upgrade logback to 1.2.3 ``PHAB_ID=D415888`` + + 19.12.0 ------- diff --git a/build.sbt b/build.sbt index 879b04a2..56377cc3 100644 --- a/build.sbt +++ b/build.sbt @@ -255,8 +255,8 @@ lazy val twitterServerSlf4jLogbackClassic = (project in file("logback-classic")) sharedSettings) .settings( libraryDependencies ++= Seq( - "ch.qos.logback" % "logback-classic" % "1.1.7" % "provided", - "ch.qos.logback" % "logback-core" % "1.1.7" % "provided", + "ch.qos.logback" % "logback-classic" % "1.2.3" % "provided", + "ch.qos.logback" % "logback-core" % "1.2.3" % "provided", "org.slf4j" % "slf4j-api" % slf4jVersion, "org.slf4j" % "jcl-over-slf4j" % slf4jVersion, "org.slf4j" % "jul-to-slf4j" % slf4jVersion, From 6e3846ffbc7a3e1b49f57708ddc193fbedf6f731 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Sun, 29 Dec 2019 03:10:11 +0000 Subject: [PATCH 128/409] Twitter-oss: update OSS libraries post-release to 20.1.0 We want to update to the next SNAPSHOT version of our Twitter OSS libraries 20.1.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. JIRA Issues: CSL-9268 Differential Revision: https://phabricator.twitter.biz/D418605 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7ff7d5c9..5baf251e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 56377cc3..dab30384 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.12.0" +val releaseVersion = "20.1.0-SNAPSHOT" val jacksonVersion = "2.9.9" val jacksonDatabindVersion = "2.9.10.1" From e03ab42b871893360e605abb8126b72044f2cc8e Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Wed, 29 Jan 2020 05:13:36 +0000 Subject: [PATCH 129/409] Twitter-oss: Prepare OSS libraries for release 20.1.0 Problem We want to release the next versions of our Twitter OSS libraries 20.1.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D426839 --- CHANGELOG.rst | 3 +++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e9e9b5cf..35ee9461 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +20.1.0 +------ + Changed ~~~~~~~ diff --git a/README.md b/README.md index 5baf251e..7ff7d5c9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index dab30384..384c83fe 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.1.0-SNAPSHOT" +val releaseVersion = "20.1.0" val jacksonVersion = "2.9.9" val jacksonDatabindVersion = "2.9.10.1" From 69ece279df9d177c811f6175baa30c23a1dbdb3f Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Wed, 29 Jan 2020 23:15:56 +0000 Subject: [PATCH 130/409] Twitter-oss: update OSS libraries post-release to 20.2.0 Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 20.2.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D427350 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7ff7d5c9..5baf251e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 384c83fe..d5b90675 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.1.0" +val releaseVersion = "20.2.0-SNAPSHOT" val jacksonVersion = "2.9.9" val jacksonDatabindVersion = "2.9.10.1" From 7093858809146e4aa01421141637708b3a86a591 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Mon, 3 Feb 2020 23:28:40 +0000 Subject: [PATCH 131/409] twitter-server: Add notes for AdminHttpServer routes changes Problem It is not pointed out some routes accessor changes in September 2019. See https://github.com/twitter/twitter-server/issues/61 for details. Solution Patch a CHANGELOG entry for these changes and add some scaladoc. JIRA Issues: CSL-9316 Differential Revision: https://phabricator.twitter.biz/D429283 --- CHANGELOG.rst | 5 ++++- .../main/scala/com/twitter/server/AdminHttpServer.scala | 9 +++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 35ee9461..7683911a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -54,7 +54,10 @@ No Changes 19.9.0 ------ -No Changes +* Changed `com.twitter.server.AdminHttpServer.routes` from a setter to getter, use it to get + all admin http server Routes. Use `com.twitter.server.AdminHttpServer.addAdminRoute` to add + a Route and `com.twitter.server.AdminHttpServer.addAdminRoutes` to add many Routes. + ``PHAB_ID=D354013`` 19.8.0 ------ diff --git a/server/src/main/scala/com/twitter/server/AdminHttpServer.scala b/server/src/main/scala/com/twitter/server/AdminHttpServer.scala index 0ee92dd3..cd5bfd68 100644 --- a/server/src/main/scala/com/twitter/server/AdminHttpServer.scala +++ b/server/src/main/scala/com/twitter/server/AdminHttpServer.scala @@ -150,15 +150,24 @@ trait AdminHttpServer { self: App with Stats => adminHttpServer.boundAddress.asInstanceOf[InetSocketAddress] } + /** + * Add a collection of [[Route]]s into admin http server. + */ def addAdminRoutes(newRoutes: Seq[Route]): Unit = synchronized { allRoutes = allRoutes ++ newRoutes updateMuxer() } + /** + * Add a [[Route]] into admin http server. + */ def addAdminRoute(route: Route): Unit = { addAdminRoutes(Seq(route)) } + /** + * Get all [[Route]]s of admin http server. + */ def routes: Seq[Route] = allRoutes /** From 9a936b0248e26fd9fc239af6436e25e9c96efac8 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Thu, 13 Feb 2020 00:54:59 +0000 Subject: [PATCH 132/409] util-app,twitter-server,inject-app,inject-server: Better Java support Problem We are missing an easy way for Java users to specify `closeOnExitLast` functions in addition to having any Java App/TwitterServer lifecycle ergonomics in the Finatra `c.t.inject.app.AbstractApp` or the `c.t.inject.server.AbstractTwitterServer`. Right now Java users cannot easily specify a last exit function and Finatra Java users do not have access to the same basic lifecycle functions as are exposed in the `c.t.server.AbstractTwitterServer`. Solution Introduce `c.t.util.app.App#onExitLast` and carry it through to `c.t.server.TwitterServer`. Expose Java hooks in `c.t.server.AbstractTwitterServer` as well as `c.t.inject.app.AbstractApp` and `c.t.inject.server.AbstractTwitterServer` for Finatra users. Result Better support Java users around the base App and TwitterServer lifecycle. JIRA Issues: CSL-9402 Differential Revision: https://phabricator.twitter.biz/D433874 --- CHANGELOG.rst | 3 + .../com/twitter/server/TwitterServer.scala | 6 ++ .../server/TwitterServerCompilationTest.java | 13 +++-- .../twitter/server/TwitterServerTest.scala | 58 ++++++++++++++++--- 4 files changed, 68 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7683911a..2506785e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +* Add `c.t.server.AbstractTwitterServer#onExitLast` to allow Java users to + easily register a final exit function. ``PHAB_ID=D433874`` + 20.1.0 ------ diff --git a/server/src/main/scala/com/twitter/server/TwitterServer.scala b/server/src/main/scala/com/twitter/server/TwitterServer.scala index 6d248fad..514b6f25 100644 --- a/server/src/main/scala/com/twitter/server/TwitterServer.scala +++ b/server/src/main/scala/com/twitter/server/TwitterServer.scala @@ -96,6 +96,11 @@ abstract class AbstractTwitterServer extends TwitterServer { */ def onExit(): Unit = () + /** + * Called prior to application exiting after `onExit`. + */ + def onExitLast(): Unit = () + /** * The `main` method of this server. */ @@ -106,4 +111,5 @@ abstract class AbstractTwitterServer extends TwitterServer { premain(preMain()) postmain(postMain()) onExit(onExit()) + onExitLast(onExitLast()) } diff --git a/server/src/test/java/com/twitter/server/TwitterServerCompilationTest.java b/server/src/test/java/com/twitter/server/TwitterServerCompilationTest.java index 35b1395a..2474c220 100644 --- a/server/src/test/java/com/twitter/server/TwitterServerCompilationTest.java +++ b/server/src/test/java/com/twitter/server/TwitterServerCompilationTest.java @@ -12,6 +12,11 @@ public class TwitterServerCompilationTest extends AbstractTwitterServer { private Flag foo = flag().create("foo", "default-foo", "help-foo", Flaggable.ofString()); + @Override + public void onInit() { + LOG.info("on-init"); + } + @Override public void preMain() { LOG.info("pre-exit"); @@ -36,12 +41,12 @@ public void postMain() { } @Override - public void onInit() { - LOG.info("on-init"); + public void onExit() { + LOG.info("on-exit"); } @Override - public void onExit() { - LOG.info("on-exit"); + public void onExitLast() { + LOG.info("on-exit-last"); } } diff --git a/server/src/test/scala/com/twitter/server/TwitterServerTest.scala b/server/src/test/scala/com/twitter/server/TwitterServerTest.scala index 337c2c28..81953ee6 100644 --- a/server/src/test/scala/com/twitter/server/TwitterServerTest.scala +++ b/server/src/test/scala/com/twitter/server/TwitterServerTest.scala @@ -1,21 +1,30 @@ package com.twitter.server +import com.twitter.concurrent.NamedPoolThreadFactory import com.twitter.conversions.DurationOps._ import com.twitter.finagle.Service import com.twitter.finagle.http.{Request, Response} import com.twitter.util._ +import java.util.concurrent.Executors import org.scalatest.FunSuite import scala.collection.mutable +import scala.util.control.NonFatal -class TestTwitterServer extends TwitterServer { +class TestTwitterServer(await: Boolean = false) extends TwitterServer { override val defaultAdminPort = 0 /* ensure enough time to close resources */ override val defaultCloseGracePeriod: Duration = 30.seconds val bootstrapSeq: mutable.MutableList[Symbol] = mutable.MutableList.empty[Symbol] + val value: Promise[Unit] = new Promise[Unit] def main(): Unit = { bootstrapSeq += 'Main + if (await) { + value.setDone() + // block to mirror listening server await + Await.ready(Future.never) + } } init { @@ -26,12 +35,16 @@ class TestTwitterServer extends TwitterServer { bootstrapSeq += 'PreMain } + postmain { + bootstrapSeq += 'PostMain + } + onExit { bootstrapSeq += 'Exit } - postmain { - bootstrapSeq += 'PostMain + onExitLast { + bootstrapSeq += 'ExitLast } } @@ -43,21 +56,21 @@ class MockExceptionHandler extends Service[Request, Response] { } class TwitterServerTest extends FunSuite { test("TwitterServer does not prematurely execute lifecycle hooks") { - val twitterServer = new TestTwitterServer + val twitterServer = new TestTwitterServer() assert(twitterServer.bootstrapSeq.isEmpty) } test("TwitterServer.main(args) executes without error") { - val twitterServer = new TestTwitterServer + val twitterServer = new TestTwitterServer() twitterServer.main(args = Array.empty[String]) assert( twitterServer.bootstrapSeq == - Seq('Init, 'PreMain, 'Main, 'PostMain, 'Exit) + Seq('Init, 'PreMain, 'Main, 'PostMain, 'Exit, 'ExitLast) ) } test("TwitterServer.main(args) executes without error when closed explicitly") { - val twitterServer = new TestTwitterServer { + val twitterServer = new TestTwitterServer() { override def main(): Unit = { super.main() Await.result(close(), 5.seconds) @@ -65,6 +78,35 @@ class TwitterServerTest extends FunSuite { } twitterServer.main(args = Array.empty[String]) - assert(twitterServer.bootstrapSeq == Seq('Init, 'PreMain, 'Main, 'Exit, 'PostMain)) + // PostMain happens last here because close() is called in the main above which calls onExit and onExitLast + assert(twitterServer.bootstrapSeq == Seq('Init, 'PreMain, 'Main, 'Exit, 'ExitLast, 'PostMain)) + } + + test("TwitterServer.main(args) executes without error when blocking and closed") { + val twitterServer = new TestTwitterServer(await = true) + val pool = new ExecutorServiceFuturePool( + Executors.newCachedThreadPool(new NamedPoolThreadFactory("Test " + getClass.getSimpleName))) + + try { + // this will block indefinitely + pool { + twitterServer.main(args = Array.empty[String]) + } + + val close = + twitterServer.value + .flatMap(_ => twitterServer.close(twitterServer.defaultCloseGracePeriod)) + Await.result(close, 5.seconds) + // note we do not get to the PostMain because the server is blocking and + // closed outside of the main method. exit blocks are called explicitly by close() + // thus postMains are effectively skipped. + assert(twitterServer.bootstrapSeq == Seq('Init, 'PreMain, 'Main, 'Exit, 'ExitLast)) + } finally { + try { + pool.executor.shutdown() + } catch { + case NonFatal(_) => // do nothing + } + } } } From db0781378545e33f4d24bf9f522e968090ef3b75 Mon Sep 17 00:00:00 2001 From: pliu Date: Wed, 19 Feb 2020 21:27:08 +0000 Subject: [PATCH 133/409] Update BUILD files - deprecate [rz]globs `pants 1.25.x` will be deprecating `globs()`, `rglobs()`, and `zglobs()` in BUILD files This commit updates about 3K BUILD files - replaces [rz]globs. JIRA Issues: DPB-12073 Differential Revision: https://phabricator.twitter.biz/D436777 --- opencensus/src/main/scala/com/twitter/server/opencensus/BUILD | 2 +- slf4j-jdk14/src/main/scala/BUILD | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/opencensus/src/main/scala/com/twitter/server/opencensus/BUILD b/opencensus/src/main/scala/com/twitter/server/opencensus/BUILD index 909cd8a2..c8b18d35 100644 --- a/opencensus/src/main/scala/com/twitter/server/opencensus/BUILD +++ b/opencensus/src/main/scala/com/twitter/server/opencensus/BUILD @@ -1,5 +1,5 @@ scala_library( - sources = globs("*.scala"), + sources = ["*.scala"], compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/io/opencensus:opencensus-api", diff --git a/slf4j-jdk14/src/main/scala/BUILD b/slf4j-jdk14/src/main/scala/BUILD index ed275eff..848076c3 100644 --- a/slf4j-jdk14/src/main/scala/BUILD +++ b/slf4j-jdk14/src/main/scala/BUILD @@ -1,5 +1,5 @@ scala_library( - sources = rglobs("*.scala"), + sources = ["**/*.scala"], compiler_option_sets = {"fatal_warnings"}, provides = scala_artifact( org = "com.twitter", From 26f66b48954e7625524c6c569fdcc16720ef8bff Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Fri, 21 Feb 2020 16:54:27 +0000 Subject: [PATCH 134/409] Twitter-oss: Prepare OSS libraries for release 20.2.0 Problem We want to release the next versions of our Twitter OSS libraries 20.2.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-9397 Differential Revision: https://phabricator.twitter.biz/D438131 --- CHANGELOG.rst | 3 +++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 2506785e..5b72d853 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +20.2.0 +------ + * Add `c.t.server.AbstractTwitterServer#onExitLast` to allow Java users to easily register a final exit function. ``PHAB_ID=D433874`` diff --git a/README.md b/README.md index 5baf251e..7ff7d5c9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index d5b90675..d90710f5 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.2.0-SNAPSHOT" +val releaseVersion = "20.2.0" val jacksonVersion = "2.9.9" val jacksonDatabindVersion = "2.9.10.1" From 11153748123cf37049677085c5894d1e7a3f3c1a Mon Sep 17 00:00:00 2001 From: pliu Date: Fri, 21 Feb 2020 22:07:13 +0000 Subject: [PATCH 135/409] Update Build files - deprecate [rz]globs (batch#3) `pants 1.25.x` is deprecating `globs()`, `rglobs()`, and `zglobs()` in BUILD files This commit updates about 3K+ BUILD files - replaces [rz]globs. (batch#3) JIRA Issues: DPB-12073 Differential Revision: https://phabricator.twitter.biz/D437945 --- logback-classic/src/main/scala/BUILD | 2 +- logback-classic/src/test/scala/BUILD | 2 +- slf4j-jdk14/src/test/scala/BUILD | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/logback-classic/src/main/scala/BUILD b/logback-classic/src/main/scala/BUILD index eb95a09a..9ebd702a 100644 --- a/logback-classic/src/main/scala/BUILD +++ b/logback-classic/src/main/scala/BUILD @@ -1,5 +1,5 @@ scala_library( - sources = rglobs("*.scala"), + sources = ["**/*.scala"], compiler_option_sets = {"fatal_warnings"}, provides = scala_artifact( org = "com.twitter", diff --git a/logback-classic/src/test/scala/BUILD b/logback-classic/src/test/scala/BUILD index 09a13a20..e5e098b5 100644 --- a/logback-classic/src/test/scala/BUILD +++ b/logback-classic/src/test/scala/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = rglobs("*.scala"), + sources = ["**/*.scala"], compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/ch/qos/logback:logback-classic", diff --git a/slf4j-jdk14/src/test/scala/BUILD b/slf4j-jdk14/src/test/scala/BUILD index b9866b6d..eec50c3a 100644 --- a/slf4j-jdk14/src/test/scala/BUILD +++ b/slf4j-jdk14/src/test/scala/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = rglobs("*.scala"), + sources = ["**/*.scala"], compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", From 905d5ffde1fb2406f14d7fc15f767323912ad101 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Sat, 22 Feb 2020 00:58:59 +0000 Subject: [PATCH 136/409] Twitter-oss: update OSS libraries post-release to 20.3.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 20.3.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D438992 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7ff7d5c9..5baf251e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index d90710f5..8ead4766 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.2.0" +val releaseVersion = "20.3.0-SNAPSHOT" val jacksonVersion = "2.9.9" val jacksonDatabindVersion = "2.9.10.1" From a3ad560cfa73f60cbf402be10f3c04f536acb1f8 Mon Sep 17 00:00:00 2001 From: pliu Date: Wed, 26 Feb 2020 01:36:45 +0000 Subject: [PATCH 137/409] Update Build files - deprecate [rz]globs (batch#8) `pants 1.25.x` is deprecating `globs()`, `rglobs()`, and `zglobs()` in BUILD files This commit updates about 3K+ BUILD files - replaces [rz]globs. (batch#8) JIRA Issues: DPB-12073 Differential Revision: https://phabricator.twitter.biz/D440569 --- server/src/main/scala/BUILD | 2 +- server/src/test/java/BUILD | 2 +- server/src/test/scala/BUILD | 2 +- slf4j-log4j12/src/main/scala/BUILD | 2 +- slf4j-log4j12/src/test/scala/BUILD | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/src/main/scala/BUILD b/server/src/main/scala/BUILD index d6135845..51ceecf7 100644 --- a/server/src/main/scala/BUILD +++ b/server/src/main/scala/BUILD @@ -1,5 +1,5 @@ scala_library( - sources = rglobs("*.scala"), + sources = ["**/*.scala"], compiler_option_sets = {"fatal_warnings"}, provides = scala_artifact( org = "com.twitter", diff --git a/server/src/test/java/BUILD b/server/src/test/java/BUILD index f61eb507..add1710d 100644 --- a/server/src/test/java/BUILD +++ b/server/src/test/java/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = rglobs("*.java"), + sources = ["**/*.java"], compiler_option_sets = {}, dependencies = [ "3rdparty/jvm/junit", diff --git a/server/src/test/scala/BUILD b/server/src/test/scala/BUILD index 39a99a4e..7e988c97 100644 --- a/server/src/test/scala/BUILD +++ b/server/src/test/scala/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = rglobs("*.scala"), + sources = ["**/*.scala"], compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", diff --git a/slf4j-log4j12/src/main/scala/BUILD b/slf4j-log4j12/src/main/scala/BUILD index 9e8843aa..618400f2 100644 --- a/slf4j-log4j12/src/main/scala/BUILD +++ b/slf4j-log4j12/src/main/scala/BUILD @@ -1,5 +1,5 @@ scala_library( - sources = rglobs("*.scala"), + sources = ["**/*.scala"], compiler_option_sets = {"fatal_warnings"}, provides = scala_artifact( org = "com.twitter", diff --git a/slf4j-log4j12/src/test/scala/BUILD b/slf4j-log4j12/src/test/scala/BUILD index 8a0ab6ef..650695c6 100644 --- a/slf4j-log4j12/src/test/scala/BUILD +++ b/slf4j-log4j12/src/test/scala/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = rglobs("*.scala"), + sources = ["**/*.scala"], compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", From d1191d37e0e9996420051807398372f3e40b0e6f Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Wed, 26 Feb 2020 17:08:08 +0000 Subject: [PATCH 138/409] Twitter-oss: Prepare OSS libraries for release 20.2.1 Problem We want to release the next versions of our Twitter OSS libraries 20.2.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-9397 Differential Revision: https://phabricator.twitter.biz/D440817 --- CHANGELOG.rst | 2 +- README.md | 2 +- build.sbt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 5b72d853..a5d20c31 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,7 +7,7 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- -20.2.0 +20.2.1 ------ * Add `c.t.server.AbstractTwitterServer#onExitLast` to allow Java users to diff --git a/README.md b/README.md index 5baf251e..7ff7d5c9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 8ead4766..eeec17e3 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.3.0-SNAPSHOT" +val releaseVersion = "20.2.1" val jacksonVersion = "2.9.9" val jacksonDatabindVersion = "2.9.10.1" From fe758e44059a2dcc7a5e3da595d110a6bd5f3c7c Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Thu, 27 Feb 2020 00:39:58 +0000 Subject: [PATCH 139/409] Twitter-oss: update OSS libraries post-release to 20.3.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 20.3.0-SNAPSHOT util scrooge finagle twitter-server finatra Solution Prepare libraries for their next SNAPSHOT version. JIRA Issues: CSL-9397 Differential Revision: https://phabricator.twitter.biz/D441416 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7ff7d5c9..5baf251e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index eeec17e3..8ead4766 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.2.1" +val releaseVersion = "20.3.0-SNAPSHOT" val jacksonVersion = "2.9.9" val jacksonDatabindVersion = "2.9.10.1" From 17803f4a7e433194cbaaeb2a80d752ecafd310b9 Mon Sep 17 00:00:00 2001 From: pliu Date: Fri, 28 Feb 2020 01:00:40 +0000 Subject: [PATCH 140/409] Update Build files - deprecate [rz]globs (batch#11) `pants 1.25.x` is deprecating `globs()`, `rglobs()`, and `zglobs()` in BUILD files This commit updates about 3K+ BUILD files - replaces [rz]globs. (batch#11) JIRA Issues: DPB-12073 Differential Revision: https://phabricator.twitter.biz/D441822 --- logback-classic/src/main/resources/BUILD | 9 +++++---- server/src/main/resources/BUILD | 9 +++++---- server/src/test/resources/BUILD | 9 +++++---- slf4j-jdk14/src/main/resources/BUILD | 9 +++++---- slf4j-log4j12/src/main/resources/BUILD | 9 +++++---- 5 files changed, 25 insertions(+), 20 deletions(-) diff --git a/logback-classic/src/main/resources/BUILD b/logback-classic/src/main/resources/BUILD index e0afa86a..45a90181 100644 --- a/logback-classic/src/main/resources/BUILD +++ b/logback-classic/src/main/resources/BUILD @@ -1,6 +1,7 @@ resources( - sources = rglobs( - "*", - exclude = [rglobs("BUILD*", "*.pyc")], - ), + sources = [ + "!**/*.pyc", + "!BUILD*", + "**/*", + ], ) diff --git a/server/src/main/resources/BUILD b/server/src/main/resources/BUILD index e0afa86a..45a90181 100644 --- a/server/src/main/resources/BUILD +++ b/server/src/main/resources/BUILD @@ -1,6 +1,7 @@ resources( - sources = rglobs( - "*", - exclude = [rglobs("BUILD*", "*.pyc")], - ), + sources = [ + "!**/*.pyc", + "!BUILD*", + "**/*", + ], ) diff --git a/server/src/test/resources/BUILD b/server/src/test/resources/BUILD index e0afa86a..45a90181 100644 --- a/server/src/test/resources/BUILD +++ b/server/src/test/resources/BUILD @@ -1,6 +1,7 @@ resources( - sources = rglobs( - "*", - exclude = [rglobs("BUILD*", "*.pyc")], - ), + sources = [ + "!**/*.pyc", + "!BUILD*", + "**/*", + ], ) diff --git a/slf4j-jdk14/src/main/resources/BUILD b/slf4j-jdk14/src/main/resources/BUILD index e0afa86a..45a90181 100644 --- a/slf4j-jdk14/src/main/resources/BUILD +++ b/slf4j-jdk14/src/main/resources/BUILD @@ -1,6 +1,7 @@ resources( - sources = rglobs( - "*", - exclude = [rglobs("BUILD*", "*.pyc")], - ), + sources = [ + "!**/*.pyc", + "!BUILD*", + "**/*", + ], ) diff --git a/slf4j-log4j12/src/main/resources/BUILD b/slf4j-log4j12/src/main/resources/BUILD index e0afa86a..45a90181 100644 --- a/slf4j-log4j12/src/main/resources/BUILD +++ b/slf4j-log4j12/src/main/resources/BUILD @@ -1,6 +1,7 @@ resources( - sources = rglobs( - "*", - exclude = [rglobs("BUILD*", "*.pyc")], - ), + sources = [ + "!**/*.pyc", + "!BUILD*", + "**/*", + ], ) From 910266c8b79bdf8d643f3a9ccbbe272def5c8909 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Fri, 28 Feb 2020 22:40:33 +0000 Subject: [PATCH 141/409] csl: Update build.sbt configurations to use overWriting Problem When we try to resume a failed release, if it fails midway through publishing a library, it's unable to continue because it gets stuck in a retry loop trying to republish JARs. Solution We should enable overwriting so that we're able to resume publishing successfully. JIRA Issues: CSL-9476 Differential Revision: https://phabricator.twitter.biz/D442363 --- build.sbt | 1 + 1 file changed, 1 insertion(+) diff --git a/build.sbt b/build.sbt index 8ead4766..79a674ed 100644 --- a/build.sbt +++ b/build.sbt @@ -131,6 +131,7 @@ lazy val sharedSettings = Seq( publishArtifact in Test := false, pomIncludeRepository := { _ => false }, publishMavenStyle := true, + publishConfiguration := publishConfiguration.value.withOverwrite(true), autoAPIMappings := true, apiURL := Some(url("https://twitter.github.io/twitter-server/docs/")), pomExtra := From 9a1d998c35f42ee276a23c00edec5ed730ba6450 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Tue, 3 Mar 2020 23:44:18 +0000 Subject: [PATCH 142/409] util|scrooge|finagle: Update to Scala 2.13.1 Problem/Solution Update util, scrooge, and finagle to all cross build with Scala 2.13.1. We need to also update the `scoverage` plugin here to deal with an issue where plugins need to be specified to be fully cross versioned (that is published with their full scala version, e.g., `2.13.0` instead of just `2.13`) because Scala is not binary compatible between minor releases and something changed between 2.13.0 and 2.13.1. The `scoverage` plugin version `1.6.1` appears to work properly. Differential Revision: https://phabricator.twitter.biz/D444211 --- project/plugins.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/plugins.sbt b/project/plugins.sbt index 470229ff..b095855a 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -5,5 +5,5 @@ resolvers ++= Seq( addSbtPlugin("com.typesafe.sbt" % "sbt-site" % "1.3.0") addSbtPlugin("io.get-coursier" % "sbt-coursier" % "1.0.0-RC13") -addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.5.1") +addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.6.1") addSbtPlugin("com.eed3si9n" % "sbt-unidoc" % "0.4.1") From e31a6c25cf3f7d8af07e212135bd814aece47e5c Mon Sep 17 00:00:00 2001 From: Felix Bruckmeier Date: Wed, 4 Mar 2020 03:07:35 +0000 Subject: [PATCH 143/409] twitter-server: Crossbuild with Scala 2.13.1 Problem TwitterServer is not cross built with Scala 2.13.x. Solution Update the code, sbt build and TravisCI configuration to allow for crossbuilding with Scala 2.13.1. Result TwitterServer will be cross built for Scala 2.13.1 and a compatible artifact will be published upon the next release. Signed-off-by: Christopher Coco Differential Revision: https://phabricator.twitter.biz/D444101 --- .travis.yml | 8 ++++ build.sbt | 2 +- .../logback/classic/LoggingHandler.scala | 2 +- .../logback/classic/LoggingHandlerTest.scala | 2 +- .../scala/com/twitter/server/Lifecycle.scala | 2 +- .../handler/HistogramQueryHandler.scala | 4 +- .../server/handler/ToggleHandler.scala | 12 +++--- .../server/handler/TunableHandler.scala | 2 +- .../scala/com/twitter/server/HookTest.scala | 2 +- .../twitter/server/TwitterServerTest.scala | 2 +- .../handler/MetricTypeQueryHandlerTest.scala | 41 ++++++++++++++----- .../server/handler/RegistryHandlerTest.scala | 36 +++++++++++++--- .../server/handler/ResourceHandlerTest.scala | 5 ++- .../server/handler/TunableHandlerTest.scala | 30 +++++++++++++- .../com/twitter/server/HandlersTest.scala | 2 +- 15 files changed, 118 insertions(+), 34 deletions(-) diff --git a/.travis.yml b/.travis.yml index b57a2daf..cec2ac12 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,11 +26,19 @@ before_cache: scala: - 2.11.12 - 2.12.8 + - 2.13.1 jdk: - openjdk8 - openjdk11 +matrix: + # Does not allow for assigning a build result if the non-'allow_failures' targets are complete + # because of https://travis-ci.community/t/receiving-too-many-emails/4888 + fast_finish: false + allow_failures: + - scala: 2.13.1 + notifications: slack: secure: FfzYHh8IvtBaYswh9mIQU9yQGr9YpkRaFR77RaYdK5wVhw3UlCBtRTBdFpUn0djgg3Pl3FxEj5OyEKYQ803xxF7UQRxx298g4r5OhOzjQ4UH1lhnAg1WpIRPBL4DuWrreZFtnqvsygrdtvATZ1aLzaYtkaCV49x/t2Kr7weNfpQ= diff --git a/build.sbt b/build.sbt index 79a674ed..70cd450c 100644 --- a/build.sbt +++ b/build.sbt @@ -82,7 +82,7 @@ lazy val sharedSettings = Seq( version := releaseVersion, organization := "com.twitter", scalaVersion := "2.12.8", - crossScalaVersions := Seq("2.11.12", "2.12.8"), + crossScalaVersions := Seq("2.11.12", "2.12.8", "2.13.1"), fork in Test := true, // We have to fork to get the JavaOptions libraryDependencies ++= Seq( "org.scalacheck" %% "scalacheck" % "1.14.0" % "test", diff --git a/logback-classic/src/main/scala/com/twitter/server/handler/logback/classic/LoggingHandler.scala b/logback-classic/src/main/scala/com/twitter/server/handler/logback/classic/LoggingHandler.scala index ed9f05d9..ac3fe070 100644 --- a/logback-classic/src/main/scala/com/twitter/server/handler/logback/classic/LoggingHandler.scala +++ b/logback-classic/src/main/scala/com/twitter/server/handler/logback/classic/LoggingHandler.scala @@ -121,7 +121,7 @@ private class LoggingHandler extends AdminHttpMuxHandler with Logging { .filter(_.getLevel != null) .filter(_.getName != org.slf4j.Logger.ROOT_LOGGER_NAME) } else loggers - val html = renderHtml(filteredLoggers, julLoggers, message, showOverriddenOnly) + val html = renderHtml(filteredLoggers.toSeq, julLoggers, message, showOverriddenOnly) newResponse( contentType = "text/html;charset=UTF-8", diff --git a/logback-classic/src/test/scala/com/twitter/server/handler/logback/classic/LoggingHandlerTest.scala b/logback-classic/src/test/scala/com/twitter/server/handler/logback/classic/LoggingHandlerTest.scala index 798aa97a..a7bd6ed6 100644 --- a/logback-classic/src/test/scala/com/twitter/server/handler/logback/classic/LoggingHandlerTest.scala +++ b/logback-classic/src/test/scala/com/twitter/server/handler/logback/classic/LoggingHandlerTest.scala @@ -112,7 +112,7 @@ class LoggingHandlerTest extends FunSuite { test("displays all loggers when in display all mode") { val req = Request(("overridden", "false")) val html = Await.result(handler(req), 5.seconds).contentString - val loggerNames: Seq[String] = handler.loggers.map(_.getName) + val loggerNames: Seq[String] = handler.loggers.map(_.getName).toSeq for (name <- loggerNames) assert(html.contains(name)) } diff --git a/server/src/main/scala/com/twitter/server/Lifecycle.scala b/server/src/main/scala/com/twitter/server/Lifecycle.scala index b7e879b8..b4325e9a 100644 --- a/server/src/main/scala/com/twitter/server/Lifecycle.scala +++ b/server/src/main/scala/com/twitter/server/Lifecycle.scala @@ -46,7 +46,7 @@ object promoteBeforeServing object Lifecycle { private[server] class PromoteToOldGen( - runtimeArgs: Seq[String] = ManagementFactory.getRuntimeMXBean.getInputArguments.asScala) { + runtimeArgs: Seq[String] = ManagementFactory.getRuntimeMXBean.getInputArguments.asScala.toList) { private[this] val hasPromoted = new AtomicBoolean(false) private[server] def promoted: Boolean = hasPromoted.get() diff --git a/server/src/main/scala/com/twitter/server/handler/HistogramQueryHandler.scala b/server/src/main/scala/com/twitter/server/handler/HistogramQueryHandler.scala index afe88e32..f4ab9fb8 100644 --- a/server/src/main/scala/com/twitter/server/handler/HistogramQueryHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/HistogramQueryHandler.scala @@ -65,7 +65,9 @@ object HistogramQueryHandler { counts: Map[String, Seq[BucketAndCount]], transform: Seq[BucketAndCount] => Any ): String = - JsonConverter.writeToString(counts.mapValues(transform)) + // ".toMap" is important here for scala 2.13 as otherwise it will be a MapView which + // doesn't serialize correctly with Jackson + JsonConverter.writeToString(counts.mapValues(transform).toMap) // Generates html for visualizing histograms private[HistogramQueryHandler] val render: String = { diff --git a/server/src/main/scala/com/twitter/server/handler/ToggleHandler.scala b/server/src/main/scala/com/twitter/server/handler/ToggleHandler.scala index 3cbaad6f..c0a2b9a8 100644 --- a/server/src/main/scala/com/twitter/server/handler/ToggleHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/ToggleHandler.scala @@ -141,7 +141,7 @@ class ToggleHandler private[handler] (registeredLibrariesFn: () => Map[String, T if (errors.isEmpty) { newOk(getResponse(parsed)) } else { - genericResponse(Status.BadRequest, "Get failed", errors) + genericResponse(Status.BadRequest, "Get failed", errors.toSeq) } } @@ -159,7 +159,7 @@ class ToggleHandler private[handler] (registeredLibrariesFn: () => Map[String, T genericResponse( if (errors.isEmpty) Status.Ok else Status.BadRequest, if (errors.isEmpty) "Update successful" else "Update failed", - errors + errors.toSeq ) } @@ -177,7 +177,7 @@ class ToggleHandler private[handler] (registeredLibrariesFn: () => Map[String, T genericResponse( if (errors.isEmpty) Status.Ok else Status.BadRequest, if (errors.isEmpty) "Delete successful" else "Delete failed", - errors + errors.toSeq ) } @@ -260,7 +260,7 @@ class ToggleHandler private[handler] (registeredLibrariesFn: () => Map[String, T errors += s"Fraction must be [0.0-1.0], was: '$fractionStr'" } } - errors + errors.toSeq } /** @@ -276,7 +276,7 @@ class ToggleHandler private[handler] (registeredLibrariesFn: () => Map[String, T val toggleMap = registeredLibrariesFn()(libraryName) log.info(s"Deleted $libraryName's toggle $id") toggleMap.remove(id) - errors + errors.toSeq } /** package protected for testing */ @@ -338,7 +338,7 @@ class ToggleHandler private[handler] (registeredLibrariesFn: () => Map[String, T case captured: Toggle.Captured => captured.lastApply case _ => None } - LibraryToggle(Current(id, md.fraction, lastApply, md.description), details) + LibraryToggle(Current(id, md.fraction, lastApply, md.description), details.toSeq) }.toSeq } diff --git a/server/src/main/scala/com/twitter/server/handler/TunableHandler.scala b/server/src/main/scala/com/twitter/server/handler/TunableHandler.scala index fff37342..03531d31 100644 --- a/server/src/main/scala/com/twitter/server/handler/TunableHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/TunableHandler.scala @@ -140,7 +140,7 @@ class TunableHandler private[handler] (registeredIdsFn: () => Map[String, Tunabl val tunables = componentsMap.map { case (id, components) => val md = currentsMap(id) - TunableView(id, md.value.toString, components) + TunableView(id, md.value.toString, components.toSeq) }.toSeq TunableMapView(id, tunables) diff --git a/server/src/test/scala/com/twitter/server/HookTest.scala b/server/src/test/scala/com/twitter/server/HookTest.scala index 3bfd2d2b..36fe3ee1 100644 --- a/server/src/test/scala/com/twitter/server/HookTest.scala +++ b/server/src/test/scala/com/twitter/server/HookTest.scala @@ -22,7 +22,7 @@ class HookTestTwitterServer(hooks: Seq[Hook]) extends TwitterServer { /* ensure enough time to close resources */ override val defaultCloseGracePeriod: Duration = 30.seconds - val bootstrapSeq: mutable.MutableList[Symbol] = mutable.MutableList.empty[Symbol] + val bootstrapSeq: mutable.ArrayBuffer[Symbol] = mutable.ArrayBuffer.empty[Symbol] def main(): Unit = { bootstrapSeq += 'Main diff --git a/server/src/test/scala/com/twitter/server/TwitterServerTest.scala b/server/src/test/scala/com/twitter/server/TwitterServerTest.scala index 81953ee6..900ad1b0 100644 --- a/server/src/test/scala/com/twitter/server/TwitterServerTest.scala +++ b/server/src/test/scala/com/twitter/server/TwitterServerTest.scala @@ -15,7 +15,7 @@ class TestTwitterServer(await: Boolean = false) extends TwitterServer { /* ensure enough time to close resources */ override val defaultCloseGracePeriod: Duration = 30.seconds - val bootstrapSeq: mutable.MutableList[Symbol] = mutable.MutableList.empty[Symbol] + val bootstrapSeq: mutable.ArrayBuffer[Symbol] = mutable.ArrayBuffer.empty[Symbol] val value: Promise[Unit] = new Promise[Unit] def main(): Unit = { diff --git a/server/src/test/scala/com/twitter/server/handler/MetricTypeQueryHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/MetricTypeQueryHandlerTest.scala index 48dbc038..91b8a6ff 100644 --- a/server/src/test/scala/com/twitter/server/handler/MetricTypeQueryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/MetricTypeQueryHandlerTest.scala @@ -5,10 +5,26 @@ import com.twitter.finagle.stats.{InMemoryStatsReceiver, StatEntry, StatsRegistr import com.twitter.server.util.MetricSource import com.twitter.util.{Await, Duration} import java.util.concurrent.TimeUnit + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.scala.DefaultScalaModule +import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper import org.scalatest.FunSuite + import scala.collection.mutable +object MetricTypeQueryHandlerTest { + // Needs to be in companion object because of Jackson. We use `Set` to avoid ordering issues here. + case class Response(latched: Boolean, metrics: Set[Metric]) + case class Metric(name: String, kind: String) +} + class MetricTypeQueryHandlerTest extends FunSuite { + import MetricTypeQueryHandlerTest._ + + private[this] val mapper = new ObjectMapper with ScalaObjectMapper { + registerModule(DefaultScalaModule) + } case class TrialStat(delta: Double, value: Double, metricType: String) extends StatEntry @@ -45,7 +61,7 @@ class MetricTypeQueryHandlerTest extends FunSuite { // generate artificial WithHistogramDetails as per HistogramQueryHandlerTest. val histos = { val sr = new InMemoryStatsReceiver - val myStat = sr.stat("baz") + val _ = sr.stat("baz") sr } @@ -140,16 +156,19 @@ class MetricTypeQueryHandlerTest extends FunSuite { val timeout = Duration(5, TimeUnit.SECONDS) - // NOTE: these tests assume a specific iteration order over the registries - // and HashMaps which IS NOT a guarantee. should these tests begin to fail - // due to that, we will need a more robust approach to validation. - private[this] def assertJsonResponse(actual: String, expected: String) = { - assert(stripWhitespace(actual) == stripWhitespace(expected)) + /** + * Relying on the ordering of HashMaps is a bad idea and is different between 2.13 and earlier versions. + * + * This helper will try to deserialize both strings to the given type `T` before comparison, which + * avoids the ordering issue. + */ + private[this] def assertJsonResponseFor[T: Manifest](actual: String, expected: String) = { + val expectedObj = mapper.readValue[T](expected) + val actualObj = mapper.readValue[T](actual) + + assert(actualObj == expectedObj) } - private[this] def stripWhitespace(string: String): String = - string.filter { case c => !c.isWhitespace } - def testCase(latched: Boolean, request: Request): Unit = { if (request == typeRequestNoArg) { testCase(latched, request, testNameStart + "full set of metrics", responseToNoArg) @@ -181,7 +200,7 @@ class MetricTypeQueryHandlerTest extends FunSuite { | "latched" : true, """.stripMargin test(testName + " when using latched counters") { - assertJsonResponse( + assertJsonResponseFor[Response]( responseStart + responseMetrics, Await.result(latchedHandler(request), timeout).contentString) } @@ -193,7 +212,7 @@ class MetricTypeQueryHandlerTest extends FunSuite { | "latched" : false, """.stripMargin test(testName + " when using unlatched counters") { - assertJsonResponse( + assertJsonResponseFor[Response]( responseStart + responseMetrics, Await.result(unlatchedHandler(request), timeout).contentString) } diff --git a/server/src/test/scala/com/twitter/server/handler/RegistryHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/RegistryHandlerTest.scala index 5345501b..b213f649 100644 --- a/server/src/test/scala/com/twitter/server/handler/RegistryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/RegistryHandlerTest.scala @@ -1,10 +1,17 @@ package com.twitter.server.handler +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.scala.DefaultScalaModule +import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper import com.twitter.util.registry.{GlobalRegistry, SimpleRegistry} import org.scalatest.FunSuite class RegistryHandlerTest extends FunSuite { + private[this] val mapper = new ObjectMapper with ScalaObjectMapper { + registerModule(DefaultScalaModule) + } + private[this] val handler = new RegistryHandler() /** used for testing filtering */ @@ -17,14 +24,29 @@ class RegistryHandlerTest extends FunSuite { filterRegistry.put(Seq("1", "a", "3"), "4") filterRegistry.put(Seq("1", "b", "3"), "5") - // NOTE: these tests assume a specific iteration order over the registries + // Some of these tests assume a specific iteration order over the registries // and HashMaps which IS NOT a guarantee. should these tests begin to fail - // due to that, we will need a more robust approach to validation. + // due to that, we will need to use `assertJsonResponseFor` for them as well. private[this] def assertJsonResponse(filter: Option[String], expected: String) = { val actual = stripWhitespace(handler.jsonResponse(filter)) assert(actual == expected) } + /** + * Relying on the ordering of HashMaps is a bad idea and is different between 2.13 and earlier versions. + * + * This helper will try to deserialize both strings to the given type `T` before comparison, which + * avoids the ordering issue. + */ + private[this] def assertJsonResponseFor[T: Manifest](filter: Option[String], expected: String) = { + val actual = stripWhitespace(handler.jsonResponse(filter)) + + val expectedObj = mapper.readValue[T](expected) + val actualObj = mapper.readValue[T](actual) + + assert(actualObj == expectedObj) + } + private[this] def stripWhitespace(string: String): String = string.filter { case c => !c.isWhitespace } @@ -40,9 +62,13 @@ class RegistryHandlerTest extends FunSuite { test("RegistryHandler.jsonResponse filters with basic matches") { GlobalRegistry.withRegistry(filterRegistry) { - assertJsonResponse(Some("oof"), """{"registry":{"oof":"gah"}}""") - assertJsonResponse(Some("foo"), """{"registry":{"foo":{"bar":"baz","qux":"quux"}}}""") - assertJsonResponse(Some("foo/bar"), """{"registry":{"foo":{"bar":"baz"}}}""") + type Response = Map[String, Object] + + assertJsonResponseFor[Response](Some("oof"), """{"registry":{"oof":"gah"}}""") + assertJsonResponseFor[Response]( + Some("foo"), + """{"registry":{"foo":{"bar":"baz","qux":"quux"}}}""") + assertJsonResponseFor[Response](Some("foo/bar"), """{"registry":{"foo":{"bar":"baz"}}}""") } } diff --git a/server/src/test/scala/com/twitter/server/handler/ResourceHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/ResourceHandlerTest.scala index 56e82cae..b3d63aca 100644 --- a/server/src/test/scala/com/twitter/server/handler/ResourceHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/ResourceHandlerTest.scala @@ -14,8 +14,9 @@ class ResourceHandlerTest extends FunSuite { private[this] def await[T](a: Awaitable[T]): T = Await.result(a, 2.seconds) - private def staticResourceResolver(content: String): PartialFunction[String, InputStream] = - PartialFunction(_ => new ByteArrayInputStream(content.getBytes("UTF8"))) + private def staticResourceResolver(content: String): PartialFunction[String, InputStream] = { + case _ => new ByteArrayInputStream(content.getBytes("UTF8")) + } test("404") { val handler = new ResourceHandler("/", PartialFunction.empty) diff --git a/server/src/test/scala/com/twitter/server/handler/TunableHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/TunableHandlerTest.scala index fd7a7346..c1295ee4 100644 --- a/server/src/test/scala/com/twitter/server/handler/TunableHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/TunableHandlerTest.scala @@ -1,12 +1,27 @@ package com.twitter.server.handler +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.scala.DefaultScalaModule +import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper import com.twitter.conversions.DurationOps._ import com.twitter.finagle.http.{MediaType, Method, Request, Status} import com.twitter.util.Await import com.twitter.util.tunable.TunableMap import org.scalatest.FunSuite +object TunableHandlerTest { + // Needs to be in companion object because of Jackson. We use `Set` to avoid ordering issues here. + case class Response(id: String, tunables: Set[Tunable]) + case class Tunable(id: String, value: String, components: Set[Component]) + case class Component(source: String, value: String) +} + class TunableHandlerTest extends FunSuite { + import TunableHandlerTest._ + + private[this] val mapper = new ObjectMapper with ScalaObjectMapper { + registerModule(DefaultScalaModule) + } test("Non-{DELETE, GET, PUT} request returns 'MethodNotAllowed response") { val handler = new TunableHandler @@ -45,6 +60,19 @@ class TunableHandlerTest extends FunSuite { } } + /** + * Relying on the ordering of HashMaps is a bad idea and is different between 2.13 and earlier versions. + * + * This helper will try to deserialize both strings to the given type `T` before comparison, which + * avoids the ordering issue. + */ + private[this] def assertJsonResponseFor[T: Manifest](actual: String, expected: String) = { + val expectedObj = mapper.readValue[T](expected) + val actualObj = mapper.readValue[T](actual) + + assert(actualObj == expectedObj) + } + testIdErrorHandling(Method.Get) testJsonErrorHandling(Method.Put) @@ -98,7 +126,7 @@ class TunableHandlerTest extends FunSuite { | } | ] |}""".stripMargin - assert(resp.contentString == expected) + assertJsonResponseFor[Response](resp.contentString, expected) } test("GET: returns tunables for all ids") { diff --git a/slf4j-jdk14/src/test/scala/com/twitter/server/HandlersTest.scala b/slf4j-jdk14/src/test/scala/com/twitter/server/HandlersTest.scala index ca19980f..47f42990 100644 --- a/slf4j-jdk14/src/test/scala/com/twitter/server/HandlersTest.scala +++ b/slf4j-jdk14/src/test/scala/com/twitter/server/HandlersTest.scala @@ -13,7 +13,7 @@ import scala.collection.mutable class TestTwitterServer extends TwitterServer { override val defaultAdminPort = 0 - val bootstrapSeq: mutable.MutableList[Symbol] = mutable.MutableList.empty[Symbol] + val bootstrapSeq: mutable.ArrayBuffer[Symbol] = mutable.ArrayBuffer.empty[Symbol] def main(): Unit = { bootstrapSeq += 'Main From cde0649cee3ae02ba3d5e3b363d37824bce6fe85 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Thu, 5 Mar 2020 03:27:21 +0000 Subject: [PATCH 144/409] twitter-server: Update tests Problem/Solution Let's update tests to make them more DRY and patch the TravisCI configuration to no longer allow 2.13.1 failures since it is no longer expected that these builds should fail. Differential Revision: https://phabricator.twitter.biz/D444903 --- .travis.yml | 7 -- .../twitter/server/handler/JsonHelper.scala | 24 ++++++ .../handler/MetricTypeQueryHandlerTest.scala | 78 +++++++++---------- .../server/handler/RegistryHandlerTest.scala | 32 +++----- .../server/handler/TunableHandlerTest.scala | 43 ++++------ 5 files changed, 87 insertions(+), 97 deletions(-) create mode 100644 server/src/test/scala/com/twitter/server/handler/JsonHelper.scala diff --git a/.travis.yml b/.travis.yml index cec2ac12..390217e2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,13 +32,6 @@ jdk: - openjdk8 - openjdk11 -matrix: - # Does not allow for assigning a build result if the non-'allow_failures' targets are complete - # because of https://travis-ci.community/t/receiving-too-many-emails/4888 - fast_finish: false - allow_failures: - - scala: 2.13.1 - notifications: slack: secure: FfzYHh8IvtBaYswh9mIQU9yQGr9YpkRaFR77RaYdK5wVhw3UlCBtRTBdFpUn0djgg3Pl3FxEj5OyEKYQ803xxF7UQRxx298g4r5OhOzjQ4UH1lhnAg1WpIRPBL4DuWrreZFtnqvsygrdtvATZ1aLzaYtkaCV49x/t2Kr7weNfpQ= diff --git a/server/src/test/scala/com/twitter/server/handler/JsonHelper.scala b/server/src/test/scala/com/twitter/server/handler/JsonHelper.scala new file mode 100644 index 00000000..88c2339c --- /dev/null +++ b/server/src/test/scala/com/twitter/server/handler/JsonHelper.scala @@ -0,0 +1,24 @@ +package com.twitter.server.handler + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper + +object JsonHelper { + + /** + * Relying on the ordering of HashMaps is a bad idea and is different between 2.13 and earlier versions. + * + * This helper will try to deserialize both strings to the given type `T` before comparison, which + * avoids the ordering issue. + */ + def assertJsonResponseFor[T: Manifest]( + mapper: ObjectMapper with ScalaObjectMapper, + actual: String, + expected: String + ): Unit = { + val expectedObj = mapper.readValue[T](expected) + val actualObj = mapper.readValue[T](actual) + + assert(actualObj == expectedObj) + } +} diff --git a/server/src/test/scala/com/twitter/server/handler/MetricTypeQueryHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/MetricTypeQueryHandlerTest.scala index 91b8a6ff..999fd4d4 100644 --- a/server/src/test/scala/com/twitter/server/handler/MetricTypeQueryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/MetricTypeQueryHandlerTest.scala @@ -30,19 +30,23 @@ class MetricTypeQueryHandlerTest extends FunSuite { trait UnlatchedRegistry extends StatsRegistry { val latched = false - val stats = mutable.Map("foo" -> TrialStat(3, 4, "counter"), "bar" -> TrialStat(3, 4, "gauge")) + val stats: mutable.Map[String, TrialStat] = + mutable.Map("foo" -> TrialStat(3, 4, "counter"), "bar" -> TrialStat(3, 4, "gauge")) } trait LatchedRegistry extends StatsRegistry { val latched = true - val stats = mutable.Map("foo" -> TrialStat(3, 4, "counter"), "bar" -> TrialStat(3, 4, "gauge")) + val stats: mutable.Map[String, TrialStat] = + mutable.Map("foo" -> TrialStat(3, 4, "counter"), "bar" -> TrialStat(3, 4, "gauge")) } - val latchedStatsRegistry = new LatchedRegistry() { - override def apply(): Map[String, StatEntry] = stats.toMap + val latchedStatsRegistry: LatchedRegistry = new LatchedRegistry() { + override def apply(): Map[String, StatEntry] = { + stats.toMap + } } - val unlatchedStatsRegistry = new UnlatchedRegistry() { + val unlatchedStatsRegistry: UnlatchedRegistry = new UnlatchedRegistry() { override def apply(): Map[String, StatEntry] = stats.toMap } @@ -59,7 +63,7 @@ class MetricTypeQueryHandlerTest extends FunSuite { ) // generate artificial WithHistogramDetails as per HistogramQueryHandlerTest. - val histos = { + val histos: InMemoryStatsReceiver = { val sr = new InMemoryStatsReceiver val _ = sr.stat("baz") sr @@ -70,18 +74,19 @@ class MetricTypeQueryHandlerTest extends FunSuite { private[this] val unlatchedHandler = new MetricTypeQueryHandler(unlatchedMetricSource, details = Some(histos)) - val typeRequestNoArg = Request("http://$HOST:$PORT/admin/exp/metric_metadata") + val typeRequestNoArg: Request = Request("http://$HOST:$PORT/admin/exp/metric_metadata") - val typeRequestWithAnArg = Request("http://$HOST:$PORT/admin/exp/metric_metadata?m=bar") + val typeRequestWithAnArg: Request = Request("http://$HOST:$PORT/admin/exp/metric_metadata?m=bar") - val typeRequestWithManyArgs = Request("http://$HOST:$PORT/admin/exp/metric_metadata?m=foo&m=bar") + val typeRequestWithManyArgs: Request = Request( + "http://$HOST:$PORT/admin/exp/metric_metadata?m=foo&m=bar") - val typeRequestWithHisto = Request("http://$HOST:$PORT/admin/exp/metric_metadata?m=baz") + val typeRequestWithHisto: Request = Request("http://$HOST:$PORT/admin/exp/metric_metadata?m=baz") - val typeRequestWithHistoAndNon = Request( + val typeRequestWithHistoAndNon: Request = Request( "http://$HOST:$PORT/admin/exp/metric_metadata?m=foo&m=baz") - val responseToNoArg = + val responseToNoArg: String = """ | "metrics" : [ | { @@ -100,7 +105,7 @@ class MetricTypeQueryHandlerTest extends FunSuite { | } """.stripMargin - val responseToAnArg = + val responseToAnArg: String = """ | "metrics" : [ | { @@ -111,7 +116,7 @@ class MetricTypeQueryHandlerTest extends FunSuite { | } """.stripMargin - val responseToManyArgs = + val responseToManyArgs: String = """ | "metrics" : [ | { @@ -126,7 +131,7 @@ class MetricTypeQueryHandlerTest extends FunSuite { | } """.stripMargin - val responseToHisto = + val responseToHisto: String = """ | "metrics" : [ | { @@ -137,7 +142,7 @@ class MetricTypeQueryHandlerTest extends FunSuite { | } """.stripMargin - val responseToHistoAndNon = + val responseToHistoAndNon: String = """ | "metrics" : [ | { @@ -154,20 +159,7 @@ class MetricTypeQueryHandlerTest extends FunSuite { val testNameStart = "MetricTypeQueryHandler generates reasonable json for " - val timeout = Duration(5, TimeUnit.SECONDS) - - /** - * Relying on the ordering of HashMaps is a bad idea and is different between 2.13 and earlier versions. - * - * This helper will try to deserialize both strings to the given type `T` before comparison, which - * avoids the ordering issue. - */ - private[this] def assertJsonResponseFor[T: Manifest](actual: String, expected: String) = { - val expectedObj = mapper.readValue[T](expected) - val actualObj = mapper.readValue[T](actual) - - assert(actualObj == expectedObj) - } + val timeout: Duration = Duration(5, TimeUnit.SECONDS) def testCase(latched: Boolean, request: Request): Unit = { if (request == typeRequestNoArg) { @@ -200,7 +192,8 @@ class MetricTypeQueryHandlerTest extends FunSuite { | "latched" : true, """.stripMargin test(testName + " when using latched counters") { - assertJsonResponseFor[Response]( + JsonHelper.assertJsonResponseFor[Response]( + mapper, responseStart + responseMetrics, Await.result(latchedHandler(request), timeout).contentString) } @@ -212,7 +205,8 @@ class MetricTypeQueryHandlerTest extends FunSuite { | "latched" : false, """.stripMargin test(testName + " when using unlatched counters") { - assertJsonResponseFor[Response]( + JsonHelper.assertJsonResponseFor[Response]( + mapper, responseStart + responseMetrics, Await.result(unlatchedHandler(request), timeout).contentString) } @@ -221,15 +215,15 @@ class MetricTypeQueryHandlerTest extends FunSuite { } Seq( - testCase(true, typeRequestNoArg), - testCase(true, typeRequestWithManyArgs), - testCase(true, typeRequestWithAnArg), - testCase(true, typeRequestWithHisto), - testCase(true, typeRequestWithHistoAndNon), - testCase(false, typeRequestNoArg), - testCase(false, typeRequestWithManyArgs), - testCase(false, typeRequestWithAnArg), - testCase(false, typeRequestWithHisto), - testCase(false, typeRequestWithHistoAndNon) + testCase(latched = true, typeRequestNoArg), + testCase(latched = true, typeRequestWithManyArgs), + testCase(latched = true, typeRequestWithAnArg), + testCase(latched = true, typeRequestWithHisto), + testCase(latched = true, typeRequestWithHistoAndNon), + testCase(latched = false, typeRequestNoArg), + testCase(latched = false, typeRequestWithManyArgs), + testCase(latched = false, typeRequestWithAnArg), + testCase(latched = false, typeRequestWithHisto), + testCase(latched = false, typeRequestWithHistoAndNon) ) } diff --git a/server/src/test/scala/com/twitter/server/handler/RegistryHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/RegistryHandlerTest.scala index b213f649..497c687d 100644 --- a/server/src/test/scala/com/twitter/server/handler/RegistryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/RegistryHandlerTest.scala @@ -32,23 +32,8 @@ class RegistryHandlerTest extends FunSuite { assert(actual == expected) } - /** - * Relying on the ordering of HashMaps is a bad idea and is different between 2.13 and earlier versions. - * - * This helper will try to deserialize both strings to the given type `T` before comparison, which - * avoids the ordering issue. - */ - private[this] def assertJsonResponseFor[T: Manifest](filter: Option[String], expected: String) = { - val actual = stripWhitespace(handler.jsonResponse(filter)) - - val expectedObj = mapper.readValue[T](expected) - val actualObj = mapper.readValue[T](actual) - - assert(actualObj == expectedObj) - } - private[this] def stripWhitespace(string: String): String = - string.filter { case c => !c.isWhitespace } + string.filterNot(_.isWhitespace) test("RegistryHandler generates reasonable json") { val simple = new SimpleRegistry @@ -64,11 +49,18 @@ class RegistryHandlerTest extends FunSuite { GlobalRegistry.withRegistry(filterRegistry) { type Response = Map[String, Object] - assertJsonResponseFor[Response](Some("oof"), """{"registry":{"oof":"gah"}}""") - assertJsonResponseFor[Response]( - Some("foo"), + JsonHelper.assertJsonResponseFor[Response]( + mapper, + stripWhitespace(handler.jsonResponse(Some("oof"))), + """{"registry":{"oof":"gah"}}""") + JsonHelper.assertJsonResponseFor[Response]( + mapper, + stripWhitespace(handler.jsonResponse(Some("foo"))), """{"registry":{"foo":{"bar":"baz","qux":"quux"}}}""") - assertJsonResponseFor[Response](Some("foo/bar"), """{"registry":{"foo":{"bar":"baz"}}}""") + JsonHelper.assertJsonResponseFor[Response]( + mapper, + stripWhitespace(handler.jsonResponse(Some("foo/bar"))), + """{"registry":{"foo":{"bar":"baz"}}}""") } } diff --git a/server/src/test/scala/com/twitter/server/handler/TunableHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/TunableHandlerTest.scala index c1295ee4..b0db48da 100644 --- a/server/src/test/scala/com/twitter/server/handler/TunableHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/TunableHandlerTest.scala @@ -29,7 +29,7 @@ class TunableHandlerTest extends FunSuite { assert(Await.result(resp, 1.second).status == Status.MethodNotAllowed) } - def testJsonErrorHandling(method: Method) = { + def testJsonErrorHandling(method: Method): Unit = { test(method + ": Returns response with error if Media-Type is not JSON") { val handler = new TunableHandler val req = Request(method, "/") @@ -48,7 +48,7 @@ class TunableHandlerTest extends FunSuite { } } - def testIdErrorHandling(method: Method) = { + def testIdErrorHandling(method: Method): Unit = { test(method + ": Returns response with Status.NotFound if id in path is not registered") { val handler = new TunableHandler val req = Request(method, "/admin/tunables/foo") @@ -60,19 +60,6 @@ class TunableHandlerTest extends FunSuite { } } - /** - * Relying on the ordering of HashMaps is a bad idea and is different between 2.13 and earlier versions. - * - * This helper will try to deserialize both strings to the given type `T` before comparison, which - * avoids the ordering issue. - */ - private[this] def assertJsonResponseFor[T: Manifest](actual: String, expected: String) = { - val expectedObj = mapper.readValue[T](expected) - val actualObj = mapper.readValue[T](actual) - - assert(actualObj == expectedObj) - } - testIdErrorHandling(Method.Get) testJsonErrorHandling(Method.Put) @@ -126,7 +113,7 @@ class TunableHandlerTest extends FunSuite { | } | ] |}""".stripMargin - assertJsonResponseFor[Response](resp.contentString, expected) + JsonHelper.assertJsonResponseFor[Response](mapper, resp.contentString, expected) } test("GET: returns tunables for all ids") { @@ -195,7 +182,7 @@ class TunableHandlerTest extends FunSuite { | ] |}""".stripMargin val resp = handler(req) - assert(map(TunableMap.Key[String]("test_id"))() == Some("hello")) + assert(map(TunableMap.Key[String]("test_id"))().contains("hello")) assert(Await.result(resp, 1.second).status == Status.Ok) } @@ -216,9 +203,9 @@ class TunableHandlerTest extends FunSuite { | } | ] |}""".stripMargin - assert(map(key)() == Some("hello")) + assert(map(key)().contains("hello")) val resp = handler(req) - assert(map(key)() == Some("goodbye")) + assert(map(key)().contains("goodbye")) assert(Await.result(resp, 1.second).status == Status.Ok) } @@ -240,11 +227,11 @@ class TunableHandlerTest extends FunSuite { | } | ] |}""".stripMargin - assert(map(key1)() == Some("hello")) - assert(map(key2)() == Some("i'd better stick around")) + assert(map(key1)().contains("hello")) + assert(map(key2)().contains("i'd better stick around")) val resp = handler(req) - assert(map(key1)() == Some("goodbye")) - assert(map(key2)() == Some("i'd better stick around")) + assert(map(key1)().contains("goodbye")) + assert(map(key2)().contains("i'd better stick around")) assert(Await.result(resp, 1.second).status == Status.Ok) } @@ -266,7 +253,7 @@ class TunableHandlerTest extends FunSuite { | ] |}""".stripMargin val resp = handler(req) - assert(map(TunableMap.Key[String]("test_id"))() == None) + assert(map(key)().isEmpty) assert(Await.result(resp, 1.second).status == Status.Ok) } @@ -288,11 +275,11 @@ class TunableHandlerTest extends FunSuite { | } | ] |}""".stripMargin - assert(map(key1)() == Some("hello")) - assert(map(key2)() == Some("i'd better stick around")) + assert(map(key1)().contains("hello")) + assert(map(key2)().contains("i'd better stick around")) val resp = handler(req) - assert(map(key1)() == None) - assert(map(key2)() == Some("i'd better stick around")) + assert(map(key1)().isEmpty) + assert(map(key2)().contains("i'd better stick around")) assert(Await.result(resp, 1.second).status == Status.Ok) } } From b7397c1f96f4a13f4c5909fb75f440658d5e97c6 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Thu, 5 Mar 2020 16:55:41 +0000 Subject: [PATCH 145/409] twitter-server: Allow failures on Scala 2.13.1 TravisCI builds Problem I was a bit overzealous removing the allow failures matrix in the previous commit. We still need this as not all of finagle is Scala 2.13.1 compatible. The issue is when building the dependencies of twitter-server with Dodo, we first run an `sbt ++SCALA_VERSION update` in the repo to be built. For twitter-server this means when we try to build finagle for Scala 2.13.1 we end up attempting to resolve Scala 2.12 artifacts. The reason this is a problem is we cache in TravisCI by Scala version and thus no Scala 2.12 artifacts will be retrievable when we are building for Scala 2.13 and thus the TravisCI build fails: Solution Until all of Finagle is Scala 2.13 compatible, we will allow the 2.13.1 builds to fail in TravisCI such that the build is not marked as a complete failure or error. Differential Revision: https://phabricator.twitter.biz/D445397 --- .travis.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.travis.yml b/.travis.yml index 390217e2..cec2ac12 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,6 +32,13 @@ jdk: - openjdk8 - openjdk11 +matrix: + # Does not allow for assigning a build result if the non-'allow_failures' targets are complete + # because of https://travis-ci.community/t/receiving-too-many-emails/4888 + fast_finish: false + allow_failures: + - scala: 2.13.1 + notifications: slack: secure: FfzYHh8IvtBaYswh9mIQU9yQGr9YpkRaFR77RaYdK5wVhw3UlCBtRTBdFpUn0djgg3Pl3FxEj5OyEKYQ803xxF7UQRxx298g4r5OhOzjQ4UH1lhnAg1WpIRPBL4DuWrreZFtnqvsygrdtvATZ1aLzaYtkaCV49x/t2Kr7weNfpQ= From 5c6aeca2ab98a8d1ed8fff638482014a647e46f2 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Fri, 6 Mar 2020 17:34:25 +0000 Subject: [PATCH 146/409] Twitter-oss: Prepare OSS libraries for release 20.3.0 Problem We want to release the next versions of our Twitter OSS libraries 20.3.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D446094 --- CHANGELOG.rst | 5 +++++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a5d20c31..e8791823 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +20.3.0 +------ + +No Changes + 20.2.1 ------ diff --git a/README.md b/README.md index 5baf251e..7ff7d5c9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 70cd450c..2ebb5ce2 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.3.0-SNAPSHOT" +val releaseVersion = "20.3.0" val jacksonVersion = "2.9.9" val jacksonDatabindVersion = "2.9.10.1" From 5c8b7a7707527078c0f356838240b8263d2649a9 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Sat, 7 Mar 2020 18:47:54 +0000 Subject: [PATCH 147/409] Twitter-oss: update OSS libraries post-release to 20.4.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 20.4.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D446383 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7ff7d5c9..5baf251e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 2ebb5ce2..d7feda23 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.3.0" +val releaseVersion = "20.4.0-SNAPSHOT" val jacksonVersion = "2.9.9" val jacksonDatabindVersion = "2.9.10.1" From 6d9c87748154a981431b4582a812cf04b2c4f283 Mon Sep 17 00:00:00 2001 From: Matt Dannenberg Date: Tue, 17 Mar 2020 19:14:35 +0000 Subject: [PATCH 148/409] twitter-service: add metric_metadata endpoint to admin server Problem Finagle stats are currently storing but not exposing Metric Schemas containing useful metadata about metrics. Solution Create an endpoint (/admin/metric_metadata.json) which exposes this metadata in a JSON format, and which allows for metrics to be filtered by name with a query param "name" ala "m" in /admin/metrics.json. Result Users and tools can now query the new /admin/metric_metadata.json endpoint to obtain the stored metadata about a metric. JIRA Issues: CSL-8921 Differential Revision: https://phabricator.twitter.biz/D449149 --- .../main/scala/com/twitter/server/Admin.scala | 7 + .../handler/MetricMetadataQueryHandler.scala | 101 ++++++ .../twitter/server/util/JsonConverter.scala | 1 + .../server/util/MetricSchemaJsonModule.scala | 60 ++++ .../server/util/MetricSchemaSource.scala | 52 +++ .../MetricMetadataQueryHandlerTest.scala | 337 ++++++++++++++++++ .../util/MetricSchemaJsonModuleTest.scala | 91 +++++ .../server/util/MetricSchemaSourceTest.scala | 98 +++++ 8 files changed, 747 insertions(+) create mode 100644 server/src/main/scala/com/twitter/server/handler/MetricMetadataQueryHandler.scala create mode 100644 server/src/main/scala/com/twitter/server/util/MetricSchemaJsonModule.scala create mode 100644 server/src/main/scala/com/twitter/server/util/MetricSchemaSource.scala create mode 100644 server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala create mode 100644 server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala create mode 100644 server/src/test/scala/com/twitter/server/util/MetricSchemaSourceTest.scala diff --git a/server/src/main/scala/com/twitter/server/Admin.scala b/server/src/main/scala/com/twitter/server/Admin.scala index a81eaec0..39b3d328 100644 --- a/server/src/main/scala/com/twitter/server/Admin.scala +++ b/server/src/main/scala/com/twitter/server/Admin.scala @@ -161,6 +161,13 @@ object Admin { group = Some(Grouping.Metrics), includeInIndex = true ), + Route( + path = "/admin/metric_metadata.json", + handler = new MetricMetadataQueryHandler(), + alias = "Metric Metadata", + group = Some(Grouping.Metrics), + includeInIndex = true + ), Route( path = Path.Clients, handler = new ClientRegistryHandler(Path.Clients), diff --git a/server/src/main/scala/com/twitter/server/handler/MetricMetadataQueryHandler.scala b/server/src/main/scala/com/twitter/server/handler/MetricMetadataQueryHandler.scala new file mode 100644 index 00000000..0f358332 --- /dev/null +++ b/server/src/main/scala/com/twitter/server/handler/MetricMetadataQueryHandler.scala @@ -0,0 +1,101 @@ +package com.twitter.server.handler + +import com.twitter.finagle.Service +import com.twitter.finagle.http.{MediaType, Request, Response, Uri} +import com.twitter.io.Buf +import com.twitter.server.util.HttpUtils.newResponse +import com.twitter.server.util.{JsonConverter, MetricSchemaSource} +import com.twitter.util.Future + +/** + * A handler which accepts metrics metadata queries via http query strings and returns + * json encoded metrics with the metadata contained in their MetricSchemas, as well as an indicator + * of whether or not counters are latched. + * + * @note When passing an explicit histogram metric via ?name=, users must provide the raw histogram + * name, no percentile (eg, .p99) appended. + * + * Example Request: + * http://$HOST:$PORT/admin/metric_metadata?name=my/cool/counter&name=your/fine/gauge&name=my/only/histo + * + * Response: + * { + * "@version" : 1.0, + * "counters_latched" : false, + * "metrics" : [ + * { + * "name" : "my/cool/counter", + * "kind" : "counter", + * "source" : { + * "class": "finagle.stats.cool", + * "category": "Server", + * "process_path": "dc/role/zone/service" + * }, + * "description" : "Counts how many cools are seen", + * "unit" : "Requests", + * "verbosity": "Verbosity(default)", + * "key_indicator" : true + * }, + * { + * "name" : "your/fine/gauge", + * "kind" : "gauge", + * "source" : { + * "class": "finagle.stats.your", + * "category": "Client", + * "process_path": "dc/your_role/zone/your_service" + * }, + * "description" : "Measures how fine the downstream system is", + * "unit" : "Percentage", + * "verbosity": "Verbosity(debug)", + * "key_indicator" : false + * }, + * { + * "name" : "my/only/histo", + * "kind" : "histogram", + * "source" : { + * "class": "Unspecified", + * "category": "NoRoleSpecified", + * "process_path": "Unspecified" + * }, + * "description" : "No description provided", + * "unit" : "Unspecified", + * "verbosity": "Verbosity(default)", + * "key_indicator" : false, + * "buckets" : [ + * 0.5, + * 0.9, + * 0.99, + * 0.999, + * 0.9999 + * ] + * } + * ]} + */ +class MetricMetadataQueryHandler(source: MetricSchemaSource = new MetricSchemaSource) + extends Service[Request, Response] { + + private[this] def query(keys: Iterable[String]) = + for (k <- keys; e <- source.getSchema(k)) yield e + + def apply(req: Request): Future[Response] = { + val uri = Uri.fromRequest(req) + + val latched = source.hasLatchedCounters + val keysParam = uri.params.getAll("name") + + val metrics = + if (keysParam.isEmpty) source.schemaList + else query(keysParam) + + newResponse( + contentType = MediaType.JsonUtf8, + content = Buf.Utf8( + JsonConverter.writeToString( + Map( + "@version" -> 1.0, + "counters_latched" -> latched, + "metrics" -> metrics + ))) + ) + } +} diff --git a/server/src/main/scala/com/twitter/server/util/JsonConverter.scala b/server/src/main/scala/com/twitter/server/util/JsonConverter.scala index 71dfca9b..d53b26bb 100644 --- a/server/src/main/scala/com/twitter/server/util/JsonConverter.scala +++ b/server/src/main/scala/com/twitter/server/util/JsonConverter.scala @@ -13,6 +13,7 @@ object JsonConverter { factory.disable(JsonFactory.Feature.USE_THREAD_LOCAL_FOR_BUFFER_RECYCLING) val mapper = new ObjectMapper(factory) .registerModule(DefaultScalaModule) + .registerModule(MetricSchemaJsonModule) val printer = new DefaultPrettyPrinter printer.indentArraysWith(DefaultIndenter.SYSTEM_LINEFEED_INSTANCE) diff --git a/server/src/main/scala/com/twitter/server/util/MetricSchemaJsonModule.scala b/server/src/main/scala/com/twitter/server/util/MetricSchemaJsonModule.scala new file mode 100644 index 00000000..50ce5cf0 --- /dev/null +++ b/server/src/main/scala/com/twitter/server/util/MetricSchemaJsonModule.scala @@ -0,0 +1,60 @@ +package com.twitter.server.util + +import com.fasterxml.jackson.core.JsonGenerator +import com.fasterxml.jackson.databind.SerializerProvider +import com.fasterxml.jackson.databind.module.SimpleModule +import com.fasterxml.jackson.databind.ser.std.StdSerializer +import com.twitter.finagle.stats.{CounterSchema, GaugeSchema, HistogramSchema, MetricSchema} + +object SchemaSerializer extends StdSerializer[MetricSchema](classOf[MetricSchema]) { + + /** + * This custom serializer is used to convert MetricSchemas to JSON for the metric_metadata + * endpoint. + */ +// The impetus for a customer serializer over case class serde is: +// 1) The nested nature of MetricBuilder with MetricSchema means we do not have the ability to +// make decisions about which MetricBuilder fields to include based on MetricSchema type +// (ie, buckets should only be present for HistogramSchema). +// 2) Reshaping the MetricBuilder to be the top level JSON object removes the ability to inject +// the "kind" field based on the MetricSchema type. +// 3) The MetricBuilder class would need to be reworked to have a Source case class +// nested in it which would contain a few of the currently MetricBuilder level values. + def serialize( + metricSchema: MetricSchema, + jsonGenerator: JsonGenerator, + serializerProvider: SerializerProvider + ): Unit = { + jsonGenerator.writeStartObject() + jsonGenerator.writeStringField("name", metricSchema.metricBuilder.name.mkString("/")) + val dataType = metricSchema match { + case _: CounterSchema => "counter" + case _: GaugeSchema => "gauge" + case _: HistogramSchema => "histogram" + } + jsonGenerator.writeStringField("kind", dataType) + jsonGenerator.writeObjectFieldStart("source") + jsonGenerator.writeStringField( + "class", + metricSchema.metricBuilder.sourceClass.getOrElse("Unspecified")) + jsonGenerator.writeStringField("category", metricSchema.metricBuilder.role.toString) + jsonGenerator.writeStringField( + "process_path", + metricSchema.metricBuilder.processPath.getOrElse("Unspecified")) + jsonGenerator.writeEndObject() + jsonGenerator.writeStringField("description", metricSchema.metricBuilder.description) + jsonGenerator.writeStringField("unit", metricSchema.metricBuilder.units.toString) + jsonGenerator.writeStringField("verbosity", metricSchema.metricBuilder.verbosity.toString) + jsonGenerator.writeBooleanField("key_indicator", metricSchema.metricBuilder.keyIndicator) + if (metricSchema.isInstanceOf[HistogramSchema]) { + jsonGenerator.writeArrayFieldStart("buckets") + metricSchema.metricBuilder.percentiles.foreach(bucket => jsonGenerator.writeNumber(bucket)) + jsonGenerator.writeEndArray() + } + jsonGenerator.writeEndObject() + } +} + +object MetricSchemaJsonModule extends SimpleModule { + addSerializer(SchemaSerializer) +} diff --git a/server/src/main/scala/com/twitter/server/util/MetricSchemaSource.scala b/server/src/main/scala/com/twitter/server/util/MetricSchemaSource.scala new file mode 100644 index 00000000..864ddb40 --- /dev/null +++ b/server/src/main/scala/com/twitter/server/util/MetricSchemaSource.scala @@ -0,0 +1,52 @@ +package com.twitter.server.util + +import com.twitter.finagle.stats.{MetricSchema, SchemaRegistry} +import com.twitter.finagle.util.LoadService + +private[server] object MetricSchemaSource { + lazy val registry: Seq[SchemaRegistry] = LoadService[SchemaRegistry]() +} + +/** + * A map from stats names to [[com.twitter.finagle.stats.StatEntry StatsEntries]] + * which allows for stale StatEntries up to `refreshInterval`. + */ +private[server] class MetricSchemaSource( + registry: Seq[SchemaRegistry] = MetricSchemaSource.registry) { + + /** + * Indicates whether or not the MetricSource is using latched Counters. + * @note this relies on the fact that there is only one StatsRegistry and that it is + * the finagle implementation. + */ + lazy val hasLatchedCounters: Boolean = { + assert(registry.length > 0) + registry.head.hasLatchedCounters + } + + /** Returns the entry for `key` if it exists */ + def getSchema(key: String): Option[MetricSchema] = synchronized { + registry.map(_.schemas()).find(_.contains(key)).flatMap(_.get(key)) + } + + /** Returns all schemas */ + def schemaList(): Iterable[MetricSchema] = synchronized { + registry + .foldLeft(IndexedSeq[MetricSchema]()) { (seq, r) => + seq ++ r.schemas().values + } + } + + /** Returns true if the map contains `key` and false otherwise. */ + def contains(key: String): Boolean = synchronized { + registry.exists(_.schemas().contains(key)) + } + + /** Returns the set of stat keys. */ + def keySet: Set[String] = synchronized { + registry + .foldLeft(Set[String]()) { (set, r) => + set ++ r.schemas().keySet + } + } +} diff --git a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala new file mode 100644 index 00000000..944e4fd1 --- /dev/null +++ b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala @@ -0,0 +1,337 @@ +package com.twitter.server.handler + +import com.twitter.finagle.http.Request +import com.twitter.finagle.stats._ +import com.twitter.server.util.MetricSchemaSource +import com.twitter.util.Await +import org.scalatest.FunSuite + +class MetricMetadataQueryHandlerTest extends FunSuite { + + val schemaMap: Map[String, MetricSchema] = Map( + "my/cool/counter" -> CounterSchema( + new MetricBuilder( + keyIndicator = true, + description = "Counts how many cools are seen", + units = Requests, + role = Server, + verbosity = Verbosity.Default, + sourceClass = Some("finagle.stats.cool"), + name = Seq("my", "cool", "counter"), + processPath = Some("dc/role/zone/service"), + percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), + statsReceiver = null + )), + "your/fine/gauge" -> GaugeSchema( + new MetricBuilder( + keyIndicator = false, + description = "Measures how fine the downstream system is", + units = Percentage, + role = Client, + verbosity = Verbosity.Debug, + sourceClass = Some("finagle.stats.your"), + name = Seq("your", "fine", "gauge"), + processPath = Some("dc/your_role/zone/your_service"), + percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), + statsReceiver = null + )), + "my/only/histo" -> HistogramSchema( + new MetricBuilder( + name = Seq("my", "only", "histo"), + percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), + statsReceiver = null + )) + ) + + trait UnlatchedRegistry extends SchemaRegistry { + val hasLatchedCounters = false + override def schemas(): Map[String, MetricSchema] = schemaMap + } + + trait LatchedRegistry extends SchemaRegistry { + val hasLatchedCounters = true + override def schemas(): Map[String, MetricSchema] = schemaMap + } + + val latchedSchemaRegistry = new LatchedRegistry() {} + + val unlatchedSchemaRegistry = new UnlatchedRegistry() {} + + val latchedMetricSchemaSource = new MetricSchemaSource( + Seq(latchedSchemaRegistry) + ) + + val unlatchedMetricSchemaSource = new MetricSchemaSource( + Seq(unlatchedSchemaRegistry) + ) + + private[this] val latchedHandler = + new MetricMetadataQueryHandler(latchedMetricSchemaSource) + private[this] val unlatchedHandler = + new MetricMetadataQueryHandler(unlatchedMetricSchemaSource) + + val typeRequestNoArg = Request("http://$HOST:$PORT/admin/metric_metadata") + + val typeRequestWithAnArg = Request( + "http://$HOST:$PORT/admin/metric_metadata?name=your/fine/gauge") + + val typeRequestWithManyArgs = Request( + "http://$HOST:$PORT/admin/metric_metadata?name=my/cool/counter&name=your/fine/gauge") + + val typeRequestWithHisto = Request("http://$HOST:$PORT/admin/metric_metadata?name=my/only/histo") + + val typeRequestWithHistoAndNon = Request( + "http://$HOST:$PORT/admin/metric_metadata?name=my/cool/counter&name=my/only/histo") + + val responseToNoArg = + """ + | "metrics" : [ + | { + | "name" : "my/cool/counter", + | "kind" : "counter", + | "source" : { + | "class": "finagle.stats.cool", + | "category": "Server", + | "process_path": "dc/role/zone/service" + | }, + | "description" : "Counts how many cools are seen", + | "unit" : "Requests", + | "verbosity": "Verbosity(default)", + | "key_indicator" : true + | }, + | { + | "name" : "your/fine/gauge", + | "kind" : "gauge", + | "source" : { + | "class": "finagle.stats.your", + | "category": "Client", + | "process_path": "dc/your_role/zone/your_service" + | }, + | "description" : "Measures how fine the downstream system is", + | "unit" : "Percentage", + | "verbosity": "Verbosity(debug)", + | "key_indicator" : false + | }, + | { + | "name" : "my/only/histo", + | "kind" : "histogram", + | "source" : { + | "class": "Unspecified", + | "category": "NoRoleSpecified", + | "process_path": "Unspecified" + | }, + | "description" : "No description provided", + | "unit" : "Unspecified", + | "verbosity": "Verbosity(default)", + | "key_indicator" : false, + | "buckets" : [ + | 0.5, + | 0.9, + | 0.95, + | 0.99, + | 0.999, + | 0.9999 + | ] + | } + | ] + | } + """.stripMargin + + val responseToAnArg = + """ + | "metrics" : [ + | { + | "name" : "your/fine/gauge", + | "kind" : "gauge", + | "source" : { + | "class": "finagle.stats.your", + | "category": "Client", + | "process_path": "dc/your_role/zone/your_service" + | }, + | "description" : "Measures how fine the downstream system is", + | "unit" : "Percentage", + | "verbosity": "Verbosity(debug)", + | "key_indicator" : false + | } + | ] + | } """.stripMargin + + val responseToManyArgs = + """ + | "metrics" : [ + | { + | "name" : "my/cool/counter", + | "kind" : "counter", + | "source" : { + | "class": "finagle.stats.cool", + | "category": "Server", + | "process_path": "dc/role/zone/service" + | }, + | "description" : "Counts how many cools are seen", + | "unit" : "Requests", + | "verbosity": "Verbosity(default)", + | "key_indicator" : true + | }, + | { + | "name" : "your/fine/gauge", + | "kind" : "gauge", + | "source" : { + | "class": "finagle.stats.your", + | "category": "Client", + | "process_path": "dc/your_role/zone/your_service" + | }, + | "description" : "Measures how fine the downstream system is", + | "unit" : "Percentage", + | "verbosity": "Verbosity(debug)", + | "key_indicator" : false + | } + | ] + | } + """.stripMargin + + val responseToHisto = + """ + | "metrics" : [ + | { + | "name" : "my/only/histo", + | "kind" : "histogram", + | "source" : { + | "class": "Unspecified", + | "category": "NoRoleSpecified", + | "process_path": "Unspecified" + | }, + | "description" : "No description provided", + | "unit" : "Unspecified", + | "verbosity": "Verbosity(default)", + | "key_indicator" : false, + | "buckets" : [ + | 0.5, + | 0.9, + | 0.95, + | 0.99, + | 0.999, + | 0.9999 + | ] + | } + | ] + | } """.stripMargin + + val responseToHistoAndNon = + """ + | "metrics" : [ + | { + | "name" : "my/cool/counter", + | "kind" : "counter", + | "source" : { + | "class": "finagle.stats.cool", + | "category": "Server", + | "process_path": "dc/role/zone/service" + | }, + | "description" : "Counts how many cools are seen", + | "unit" : "Requests", + | "verbosity": "Verbosity(default)", + | "key_indicator" : true + | }, + | { + | "name" : "my/only/histo", + | "kind" : "histogram", + | "source" : { + | "class": "Unspecified", + | "category": "NoRoleSpecified", + | "process_path": "Unspecified" + | }, + | "description" : "No description provided", + | "unit" : "Unspecified", + | "verbosity": "Verbosity(default)", + | "key_indicator" : false, + | "buckets" : [ + | 0.5, + | 0.9, + | 0.95, + | 0.99, + | 0.999, + | 0.9999 + | ] + | } + | ] + | } + """.stripMargin + + val testNameStart = "MetricTypeQueryHandler generates reasonable json for " + + // NOTE: these tests assume a specific iteration order over the registries + // and HashMaps which IS NOT a guarantee. should these tests begin to fail + // due to that, we will need a more robust approach to validation. + private[this] def assertJsonResponse(actual: String, expected: String) = { + assert(stripWhitespace(actual) == stripWhitespace(expected)) + } + + private[this] def stripWhitespace(string: String): String = + string.filter { case c => !c.isWhitespace } + + def testCase(latched: Boolean, request: Request): Unit = { + if (request == typeRequestNoArg) { + testCase(latched, request, testNameStart + "full set of metrics", responseToNoArg) + } else if (request == typeRequestWithAnArg) { + testCase(latched, request, testNameStart + "a single requested metric", responseToAnArg) + } else if (request == typeRequestWithManyArgs) { + testCase(latched, request, testNameStart + "requested subset of metrics", responseToManyArgs) + } else if (request == typeRequestWithHisto) { + testCase(latched, request, testNameStart + "a single requested histogram", responseToHisto) + } else if (request == typeRequestWithHistoAndNon) { + testCase( + latched, + request, + testNameStart + "requested subset of metrics with a histogram", + responseToHistoAndNon) + } + } + + def testCase( + latched: Boolean, + request: Request, + testName: String, + responseMetrics: String + ): Unit = { + if (latched) { + val responseStart = + """ + | { + | "@version" : 1.0, + | "counters_latched" : true, + """.stripMargin + test(testName + " when using latched counters") { + assertJsonResponse( + responseStart + responseMetrics, + Await.result(latchedHandler(request)).contentString) + } + + } else { + val responseStart = + """ + | { + | "@version" : 1.0, + | "counters_latched" : false, + """.stripMargin + test(testName + " when using unlatched counters") { + assertJsonResponse( + responseStart + responseMetrics, + Await.result(unlatchedHandler(request)).contentString) + } + + } + } + + Seq( + testCase(true, typeRequestNoArg), + testCase(true, typeRequestWithManyArgs), + testCase(true, typeRequestWithAnArg), + testCase(true, typeRequestWithHisto), + testCase(true, typeRequestWithHistoAndNon), + testCase(false, typeRequestNoArg), + testCase(false, typeRequestWithManyArgs), + testCase(false, typeRequestWithAnArg), + testCase(false, typeRequestWithHisto), + testCase(false, typeRequestWithHistoAndNon) + ) +} diff --git a/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala b/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala new file mode 100644 index 00000000..aaad8f0e --- /dev/null +++ b/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala @@ -0,0 +1,91 @@ +package com.twitter.server.util + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.scala.DefaultScalaModule +import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper +import com.twitter.finagle.stats._ +import org.scalatest.FunSuite + +class MetricSchemaJsonModuleTest extends FunSuite { + + private val counterSchema = CounterSchema( + new MetricBuilder( + keyIndicator = true, + description = "Counts how many cools are seen", + units = Requests, + role = Server, + verbosity = Verbosity.Default, + sourceClass = Some("finagle.stats.cool"), + name = Seq("my", "cool", "counter"), + processPath = Some("dc/role/zone/service"), + percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), + statsReceiver = null + )) + private val gaugeSchema = GaugeSchema( + new MetricBuilder( + keyIndicator = false, + description = "Measures how fine the downstream system is", + units = Percentage, + role = Client, + verbosity = Verbosity.Debug, + sourceClass = Some("finagle.stats.your"), + name = Seq("your", "fine", "gauge"), + processPath = Some("dc/your_role/zone/your_service"), + percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), + statsReceiver = null + )) + private val histogramSchema = HistogramSchema( + new MetricBuilder( + name = Seq("my", "only", "histo"), + percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), + statsReceiver = null + )) + + private val topLevelFieldNameSet = + Set("name", "kind", "source", "description", "unit", "verbosity", "key_indicator") + + def jsonStrToMap(jsonStr: String): Map[String, Any] = { + val mapper = new ObjectMapper() with ScalaObjectMapper + mapper.registerModule(DefaultScalaModule) + mapper.readValue[Map[String, Any]](jsonStr) + } + + test("CounterGauge serializes with kind counter and the correct set of fields") { + val serializedString = JsonConverter.writeToString(counterSchema) + val jsonMap = jsonStrToMap(serializedString) + assert(jsonMap.keys == topLevelFieldNameSet) + assert( + jsonMap.get("source").get == Map( + "class" -> "finagle.stats.cool", + "category" -> "Server", + "process_path" -> "dc/role/zone/service")) + assert(jsonMap.get("kind").get == "counter") + } + + test("GaugeSchema serializes with kind gauge and the correct set of fields") { + val serializedString = JsonConverter.writeToString(gaugeSchema) + val jsonMap = jsonStrToMap(serializedString) + assert(jsonMap.keys == topLevelFieldNameSet) + assert( + jsonMap.get("source").get == Map( + "class" -> "finagle.stats.your", + "category" -> "Client", + "process_path" -> "dc/your_role/zone/your_service")) + assert(jsonMap.get("kind").get == "gauge") + } + + test( + "HistogramSchema serializes with kind histogram and the correct set of fields (includeing buckets)") { + val serializedString = JsonConverter.writeToString(histogramSchema) + val jsonMap = jsonStrToMap(serializedString) + assert(jsonMap.keys == topLevelFieldNameSet ++ Seq("buckets")) + assert( + jsonMap.get("source").get == Map( + "class" -> "Unspecified", + "category" -> "NoRoleSpecified", + "process_path" -> "Unspecified")) + assert(jsonMap.get("kind").get == "histogram") + assert(jsonMap.get("buckets").get == IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999)) + } + +} diff --git a/server/src/test/scala/com/twitter/server/util/MetricSchemaSourceTest.scala b/server/src/test/scala/com/twitter/server/util/MetricSchemaSourceTest.scala new file mode 100644 index 00000000..b4b856ce --- /dev/null +++ b/server/src/test/scala/com/twitter/server/util/MetricSchemaSourceTest.scala @@ -0,0 +1,98 @@ +package com.twitter.server.util + +import com.twitter.finagle.stats._ +import org.scalatest.FunSuite + +class MetricSchemaSourceTest extends FunSuite { + + private val schemaMap: Map[String, MetricSchema] = Map( + "my/cool/counter" -> CounterSchema( + new MetricBuilder( + keyIndicator = true, + description = "Counts how many cools are seen", + units = Requests, + role = Server, + verbosity = Verbosity.Default, + sourceClass = Some("finagle.stats.cool"), + name = Seq("my", "cool", "counter"), + processPath = Some("dc/role/zone/service"), + percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), + statsReceiver = null + )), + "your/fine/gauge" -> GaugeSchema( + new MetricBuilder( + keyIndicator = false, + description = "Measures how fine the downstream system is", + units = Percentage, + role = Client, + verbosity = Verbosity.Debug, + sourceClass = Some("finagle.stats.your"), + name = Seq("your", "fine", "gauge"), + processPath = Some("dc/your_role/zone/your_service"), + percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), + statsReceiver = null + )), + "my/only/histo" -> HistogramSchema( + new MetricBuilder( + name = Seq("my", "only", "histo"), + percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), + statsReceiver = null + )) + ) + + private val latchedPopulatedRegistry: SchemaRegistry = new SchemaRegistry { + override def hasLatchedCounters: Boolean = true + override def schemas(): Map[String, MetricSchema] = schemaMap + } + + private val unlatchedPopulatedRegistry: SchemaRegistry = new SchemaRegistry { + override def hasLatchedCounters: Boolean = false + override def schemas(): Map[String, MetricSchema] = schemaMap + } + + private val metricSchemaSource = new MetricSchemaSource(Seq(latchedPopulatedRegistry)) + private val unlatchedMetricSchemaSource = new MetricSchemaSource(Seq(unlatchedPopulatedRegistry)) + private val emptyMetricSchemaSource = new MetricSchemaSource(Seq()) + + test("hasLatchedCounters asserts if there is no SchemaRegistry") { + assertThrows[AssertionError](emptyMetricSchemaSource.hasLatchedCounters) + } + + test("hasLatchedCounters returns the underlying SchemaRegistry's hasLatchedCounters value") { + assert(metricSchemaSource.hasLatchedCounters) + assert(!unlatchedMetricSchemaSource.hasLatchedCounters) + } + + test("getSchema returns the appropriate MetricSchema when there is one") { + assert(metricSchemaSource.getSchema("my/cool/counter") == schemaMap.get("my/cool/counter")) + } + + test("getSchema returns the None when absent") { + assert(metricSchemaSource.getSchema("my/dull/counter") == None) + } + + test("schemaList returns the full list of MetricSchemas") { + assert(metricSchemaSource.schemaList() == schemaMap.values.toVector) + } + + test("schemaList returns empty list if there is no registry") { + assert(emptyMetricSchemaSource.schemaList() == Seq()) + } + + test("contains accurately reflect the presence or absence of a Metric from the MetricSchema map") { + assert(metricSchemaSource.contains("my/cool/counter")) + assert(!metricSchemaSource.contains("my/dull/counter")) + assert(metricSchemaSource.contains("my/only/histo")) + assert(!emptyMetricSchemaSource.contains("my/cool/counter")) + assert(!emptyMetricSchemaSource.contains("my/dull/counter")) + assert(!emptyMetricSchemaSource.contains("my/only/histo")) + } + + test("keySet returns Set of Metric names (key portion of the schema map)") { + assert(metricSchemaSource.keySet == schemaMap.keySet) + } + + test("keySet returns empty Set if there is no registry") { + assert(emptyMetricSchemaSource.keySet == Set()) + } +} From b888ebf2760f626a64ae3bad6cdcddd547c21ac7 Mon Sep 17 00:00:00 2001 From: Shane Delmore Date: Thu, 26 Mar 2020 23:15:29 +0000 Subject: [PATCH 149/409] twitter-server: Reformat with scalafmt 2.4.2 Differential Revision: https://phabricator.twitter.biz/D455456 --- build.sbt | 113 ++++++++---------- .../server/opencensus/ZPagesAdminRoutes.scala | 4 +- project/build.sbt | 3 +- .../com/twitter/server/AdminHttpServer.scala | 4 +- .../main/scala/com/twitter/server/Hooks.scala | 4 +- .../handler/AttachedClientsHandler.scala | 4 +- .../handler/HistogramQueryHandler.scala | 4 +- .../twitter/server/handler/IndexHandler.scala | 4 +- .../twitter/server/handler/LintHandler.scala | 4 +- .../server/handler/ToggleHandler.scala | 10 +- .../lint/DuplicateFlagDefinitions.scala | 4 +- .../lint/DuplicateLoadServiceBindings.scala | 4 +- .../twitter/server/lint/LoggingRules.scala | 4 +- .../server/util/MetricSchemaSource.scala | 8 +- .../twitter/server/util/MetricSource.scala | 8 +- .../com/twitter/server/view/ThreadsView.scala | 4 +- .../twitter/server/AdminHttpServerTest.scala | 8 +- .../handler/MetricTypeQueryHandlerTest.scala | 16 +-- .../server/handler/ShutdownHandlerTest.scala | 8 +- .../server/handler/ToggleHandlerTest.scala | 8 +- .../twitter/server/util/HtmlUtilsTest.scala | 4 +- .../server/util/MetricSourceTest.scala | 4 +- .../server/view/StackRegistryViewTest.scala | 4 +- 23 files changed, 85 insertions(+), 153 deletions(-) diff --git a/build.sbt b/build.sbt index d7feda23..f27d3dce 100644 --- a/build.sbt +++ b/build.sbt @@ -8,13 +8,13 @@ val jacksonDatabindVersion = "2.9.10.1" val jacksonLibs = Seq( "com.fasterxml.jackson.core" % "jackson-core" % jacksonVersion, "com.fasterxml.jackson.core" % "jackson-databind" % jacksonDatabindVersion, - "com.fasterxml.jackson.module" %% "jackson-module-scala" % jacksonVersion exclude("com.google.guava", "guava") + "com.fasterxml.jackson.module" %% "jackson-module-scala" % jacksonVersion exclude ("com.google.guava", "guava") ) val opencensusVersion = "0.19.1" val slf4jVersion = "1.7.30" -def util(which: String) = "com.twitter" %% ("util-"+which) % releaseVersion -def finagle(which: String) = "com.twitter" %% ("finagle-"+which) % releaseVersion +def util(which: String) = "com.twitter" %% ("util-" + which) % releaseVersion +def finagle(which: String) = "com.twitter" %% ("finagle-" + which) % releaseVersion lazy val noPublishSettings = Seq( publish := {}, @@ -91,42 +91,35 @@ lazy val sharedSettings = Seq( "com.novocode" % "junit-interface" % "0.11" % "test", "org.mockito" % "mockito-all" % "1.9.5" % "test" ), - ScoverageKeys.coverageHighlighting := true, - ivyXML := , - scalacOptions ++= Seq( "-target:jvm-1.8", "-deprecation", "-unchecked", - "-feature", "-Xlint", - "-encoding", "utf8" + "-feature", + "-Xlint", + "-encoding", + "utf8" ), javacOptions ++= Seq("-Xlint:unchecked", "-source", "1.8", "-target", "1.8"), javacOptions in doc := Seq("-source", "1.8"), - javaOptions ++= Seq( "-Djava.net.preferIPv4Stack=true", "-XX:+AggressiveOpts", "-server" ), - javaOptions ++= gcJavaOptions, - javaOptions in Test ++= travisTestJavaOptions, - // This is bad news for things like com.twitter.util.Time parallelExecution in Test := false, - // -a: print stack traces for failing asserts testOptions += Tests.Argument(TestFrameworks.JUnit, "-a"), - // Sonatype publishing publishArtifact in Test := false, pomIncludeRepository := { _ => false }, @@ -158,7 +151,7 @@ lazy val sharedSettings = Seq( if (version.value.trim.endsWith("SNAPSHOT")) Some("snapshots" at nexus + "content/repositories/snapshots") else - Some("releases" at nexus + "service/local/staging/deploy/maven2") + Some("releases" at nexus + "service/local/staging/deploy/maven2") } ) @@ -179,10 +172,7 @@ lazy val twitterServer = (project in file("server")) .enablePlugins( ScalaUnidocPlugin ) - .settings( - name := "twitter-server", - moduleName := "twitter-server", - sharedSettings) + .settings(name := "twitter-server", moduleName := "twitter-server", sharedSettings) .settings( libraryDependencies ++= Seq( "org.slf4j" % "slf4j-api" % slf4jVersion, @@ -199,15 +189,16 @@ lazy val twitterServer = (project in file("server")) util("slf4j-jul-bridge"), util("tunable") ), - libraryDependencies ++= jacksonLibs) + libraryDependencies ++= jacksonLibs + ) lazy val twitterServerOpenCensus = (project in file("opencensus")) .enablePlugins( ScalaUnidocPlugin ) .settings( - name := "twitter-server-opencensus", - moduleName := "twitter-server-opencensus", + name := "twitter-server-opencensus", + moduleName := "twitter-server-opencensus", sharedSettings) .settings( libraryDependencies ++= Seq( @@ -216,54 +207,49 @@ lazy val twitterServerOpenCensus = (project in file("opencensus")) "io.opencensus" % "opencensus-api" % opencensusVersion, "io.opencensus" % "opencensus-contrib-zpages" % opencensusVersion )) - .dependsOn( - twitterServer) - + .dependsOn(twitterServer) lazy val twitterServerSlf4jJdk14 = (project in file("slf4j-jdk14")) .settings( - name := "twitter-server-slf4j-jdk14", - moduleName := "twitter-server-slf4j-jdk14", + name := "twitter-server-slf4j-jdk14", + moduleName := "twitter-server-slf4j-jdk14", sharedSettings) - .settings( - libraryDependencies ++= Seq( - "org.slf4j" % "slf4j-api" % slf4jVersion, - "org.slf4j" % "slf4j-jdk14" % slf4jVersion, - "org.slf4j" % "jcl-over-slf4j" % slf4jVersion, - "org.slf4j" % "log4j-over-slf4j" % slf4jVersion)) - .dependsOn( - twitterServer) + .settings(libraryDependencies ++= Seq( + "org.slf4j" % "slf4j-api" % slf4jVersion, + "org.slf4j" % "slf4j-jdk14" % slf4jVersion, + "org.slf4j" % "jcl-over-slf4j" % slf4jVersion, + "org.slf4j" % "log4j-over-slf4j" % slf4jVersion + )) + .dependsOn(twitterServer) lazy val twitterServerSlf4jLog4j12 = (project in file("slf4j-log4j12")) .settings( - name := "twitter-server-slf4j-log4j12", - moduleName := "twitter-server-slf4j-log4j12", + name := "twitter-server-slf4j-log4j12", + moduleName := "twitter-server-slf4j-log4j12", sharedSettings) - .settings( - libraryDependencies ++= Seq( - "log4j" % "log4j" % "1.2.17" % "provided", - "org.slf4j" % "slf4j-api" % slf4jVersion, - "org.slf4j" % "slf4j-log4j12" % slf4jVersion, - "org.slf4j" % "jcl-over-slf4j" % slf4jVersion, - "org.slf4j" % "jul-to-slf4j" % slf4jVersion)) - .dependsOn( - twitterServer) + .settings(libraryDependencies ++= Seq( + "log4j" % "log4j" % "1.2.17" % "provided", + "org.slf4j" % "slf4j-api" % slf4jVersion, + "org.slf4j" % "slf4j-log4j12" % slf4jVersion, + "org.slf4j" % "jcl-over-slf4j" % slf4jVersion, + "org.slf4j" % "jul-to-slf4j" % slf4jVersion + )) + .dependsOn(twitterServer) lazy val twitterServerSlf4jLogbackClassic = (project in file("logback-classic")) .settings( - name := "twitter-server-logback-classic", - moduleName := "twitter-server-logback-classic", + name := "twitter-server-logback-classic", + moduleName := "twitter-server-logback-classic", sharedSettings) - .settings( - libraryDependencies ++= Seq( - "ch.qos.logback" % "logback-classic" % "1.2.3" % "provided", - "ch.qos.logback" % "logback-core" % "1.2.3" % "provided", - "org.slf4j" % "slf4j-api" % slf4jVersion, - "org.slf4j" % "jcl-over-slf4j" % slf4jVersion, - "org.slf4j" % "jul-to-slf4j" % slf4jVersion, - "org.slf4j" % "log4j-over-slf4j" % slf4jVersion)) - .dependsOn( - twitterServer) + .settings(libraryDependencies ++= Seq( + "ch.qos.logback" % "logback-classic" % "1.2.3" % "provided", + "ch.qos.logback" % "logback-core" % "1.2.3" % "provided", + "org.slf4j" % "slf4j-api" % slf4jVersion, + "org.slf4j" % "jcl-over-slf4j" % slf4jVersion, + "org.slf4j" % "jul-to-slf4j" % slf4jVersion, + "org.slf4j" % "log4j-over-slf4j" % slf4jVersion + )) + .dependsOn(twitterServer) lazy val twitterServerDoc = (project in file("doc")) .enablePlugins( @@ -275,14 +261,15 @@ lazy val twitterServerDoc = (project in file("doc")) sharedSettings, Seq( scalacOptions in doc ++= Seq("-doc-title", "TwitterServer", "-doc-version", version.value), - includeFilter in Sphinx := ("*.html" | "*.png" | "*.js" | "*.css" | "*.gif" | "*.txt"))) - .configs(DocTest).settings( - inConfig(DocTest)(Defaults.testSettings): _*) + includeFilter in Sphinx := ("*.html" | "*.png" | "*.js" | "*.css" | "*.gif" | "*.txt") + ) + ) + .configs(DocTest).settings(inConfig(DocTest)(Defaults.testSettings): _*) .settings( unmanagedSourceDirectories in DocTest += baseDirectory.value / "src/sphinx/code", - // Make the "test" command run both, test and doctest:test - test := Seq(test in Test, test in DocTest).dependOn.value) + test := Seq(test in Test, test in DocTest).dependOn.value + ) .dependsOn(twitterServer) /* Test Configuration for running tests on doc sources */ diff --git a/opencensus/src/main/scala/com/twitter/server/opencensus/ZPagesAdminRoutes.scala b/opencensus/src/main/scala/com/twitter/server/opencensus/ZPagesAdminRoutes.scala index 9bf247b5..ed6b133f 100644 --- a/opencensus/src/main/scala/com/twitter/server/opencensus/ZPagesAdminRoutes.scala +++ b/opencensus/src/main/scala/com/twitter/server/opencensus/ZPagesAdminRoutes.scala @@ -20,9 +20,7 @@ private object ZPagesAdminRoutes { def apply(request: Request): Future[Response] = { val requestParams = request - .getParamNames().asScala.map { name => - name -> request.getParam(name) - }.toMap.asJava + .getParamNames().asScala.map { name => name -> request.getParam(name) }.toMap.asJava // process in a FuturePool to handle the possibility // of zpages having blocking code. diff --git a/project/build.sbt b/project/build.sbt index 0c8c057a..26f5fb29 100644 --- a/project/build.sbt +++ b/project/build.sbt @@ -2,5 +2,6 @@ scalacOptions ++= Seq( "-deprecation", "-unchecked", "-feature", - "-encoding", "utf8" + "-encoding", + "utf8" ) diff --git a/server/src/main/scala/com/twitter/server/AdminHttpServer.scala b/server/src/main/scala/com/twitter/server/AdminHttpServer.scala index cd5bfd68..449b8d5f 100644 --- a/server/src/main/scala/com/twitter/server/AdminHttpServer.scala +++ b/server/src/main/scala/com/twitter/server/AdminHttpServer.scala @@ -220,9 +220,7 @@ trait AdminHttpServer { self: App with Stats => muxer.withHandler(route.path, service) } - val endpoints = allRoutes.map { route => - s"\t${route.path} => ${route.handler.toString}" - } + val endpoints = allRoutes.map { route => s"\t${route.path} => ${route.handler.toString}" } log.debug(s"AdminHttpServer Muxer endpoints:\n" + endpoints.mkString("\n")) adminHttpMuxer.underlying = HttpUtils.combine(Seq(localMuxer, HttpMuxer)) diff --git a/server/src/main/scala/com/twitter/server/Hooks.scala b/server/src/main/scala/com/twitter/server/Hooks.scala index 8bb2f6da..38d3c4e5 100644 --- a/server/src/main/scala/com/twitter/server/Hooks.scala +++ b/server/src/main/scala/com/twitter/server/Hooks.scala @@ -7,9 +7,7 @@ import com.twitter.finagle.util.LoadService * Mix-in to include service-loaded hooks. */ trait Hooks { self: App => - private val hooks = LoadService[NewHook]().map { newHook => - newHook(self) - } + private val hooks = LoadService[NewHook]().map { newHook => newHook(self) } premain { for (hook <- hooks) diff --git a/server/src/main/scala/com/twitter/server/handler/AttachedClientsHandler.scala b/server/src/main/scala/com/twitter/server/handler/AttachedClientsHandler.scala index 1df9c738..abc8f131 100644 --- a/server/src/main/scala/com/twitter/server/handler/AttachedClientsHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/AttachedClientsHandler.scala @@ -46,9 +46,7 @@ private[handler] object AttachedClientsHandler { sessionId = clientConnection.sslSessionInfo.sessionId, cipherSuite = clientConnection.sslSessionInfo.cipherSuite, peerCertificate = clientConnection.sslSessionInfo.peerCertificates.headOption - .map { peerCertificate => - PeerCertInfo(peerCertificate.getSubjectDN.getName) - } + .map { peerCertificate => PeerCertInfo(peerCertificate.getSubjectDN.getName) } )) } ) diff --git a/server/src/main/scala/com/twitter/server/handler/HistogramQueryHandler.scala b/server/src/main/scala/com/twitter/server/handler/HistogramQueryHandler.scala index f4ab9fb8..b0d0eeaf 100644 --- a/server/src/main/scala/com/twitter/server/handler/HistogramQueryHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/HistogramQueryHandler.scala @@ -46,9 +46,7 @@ object HistogramQueryHandler { // For each key return a percentage private[server] def pdf(counts: Seq[BucketAndCount]): Seq[BucketAndPercentage] = { val count = countPoints(counts) - counts.map { v => - BucketAndPercentage(v.lowerLimit, v.upperLimit, v.count.toFloat / count) - } + counts.map { v => BucketAndPercentage(v.lowerLimit, v.upperLimit, v.count.toFloat / count) } } // For each key return a cumulative percentage diff --git a/server/src/main/scala/com/twitter/server/handler/IndexHandler.scala b/server/src/main/scala/com/twitter/server/handler/IndexHandler.scala index ab33afaf..e07505f0 100644 --- a/server/src/main/scala/com/twitter/server/handler/IndexHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/IndexHandler.scala @@ -16,9 +16,7 @@ class IndexHandler(prefix: String = "/", patterns: Seq[String] = HttpMuxer.patte extends Service[Request, Response] { def apply(req: Request): Future[Response] = { val paths = patterns.filter(_.startsWith(prefix)) - val links = paths map { p => - s"${escapeHtml(p)}" - } + val links = paths map { p => s"${escapeHtml(p)}" } if (!expectsHtml(req)) newOk(paths.mkString("\n")) else newResponse( diff --git a/server/src/main/scala/com/twitter/server/handler/LintHandler.scala b/server/src/main/scala/com/twitter/server/handler/LintHandler.scala index 294a4e7f..9a0874f4 100644 --- a/server/src/main/scala/com/twitter/server/handler/LintHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/LintHandler.scala @@ -174,9 +174,7 @@ private object LintHandler { .map { case (rule, issues) => issues - .map { issue => - failedRow(rule, issue) - } + .map { issue => failedRow(rule, issue) } .mkString("") } .mkString("") diff --git a/server/src/main/scala/com/twitter/server/handler/ToggleHandler.scala b/server/src/main/scala/com/twitter/server/handler/ToggleHandler.scala index c0a2b9a8..f9b32cb2 100644 --- a/server/src/main/scala/com/twitter/server/handler/ToggleHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/ToggleHandler.scala @@ -289,14 +289,12 @@ class ToggleHandler private[handler] (registeredLibrariesFn: () => Map[String, T val libraryFilter: String => Boolean = parsedPath.libraryName match { case Some(name) => _ == name case None => - _ => - true + _ => true } val idFilter: String => Boolean = parsedPath.id match { case Some(name) => _ == name case None => - _ => - true + _ => true } val registered = registeredLibrariesFn() @@ -317,9 +315,7 @@ class ToggleHandler private[handler] (registeredLibrariesFn: () => Map[String, T private[this] def toLibraryToggles(toggleMap: ToggleMap): Seq[LibraryToggle] = { // create a map of id to metadata for faster lookups - val idToMetadata = toggleMap.iterator.map { md => - md.id -> md - }.toMap + val idToMetadata = toggleMap.iterator.map { md => md.id -> md }.toMap // create a mapping of id to a seq of its components. val idToComponents = mutable.Map.empty[String, mutable.ArrayBuffer[Component]] diff --git a/server/src/main/scala/com/twitter/server/lint/DuplicateFlagDefinitions.scala b/server/src/main/scala/com/twitter/server/lint/DuplicateFlagDefinitions.scala index 9abdb8af..bd7c79fe 100644 --- a/server/src/main/scala/com/twitter/server/lint/DuplicateFlagDefinitions.scala +++ b/server/src/main/scala/com/twitter/server/lint/DuplicateFlagDefinitions.scala @@ -23,8 +23,6 @@ object DuplicateFlagDefinitions { if (dupes.isEmpty) { Nil } else { - dupes.map { dupe => - Issue(s"Duplicate flag named: $dupe") - }.toSeq + dupes.map { dupe => Issue(s"Duplicate flag named: $dupe") }.toSeq } } diff --git a/server/src/main/scala/com/twitter/server/lint/DuplicateLoadServiceBindings.scala b/server/src/main/scala/com/twitter/server/lint/DuplicateLoadServiceBindings.scala index 9c4b3b0d..fcbf76a4 100644 --- a/server/src/main/scala/com/twitter/server/lint/DuplicateLoadServiceBindings.scala +++ b/server/src/main/scala/com/twitter/server/lint/DuplicateLoadServiceBindings.scala @@ -27,9 +27,7 @@ object DuplicateLoadServiceBindings { if (dupes.isEmpty) { Nil } else { - dupes.map { dupe => - Issue(s"Duplicate for interface: ${dupe.getName}") - }.toSeq + dupes.map { dupe => Issue(s"Duplicate for interface: ${dupe.getName}") }.toSeq } } diff --git a/server/src/main/scala/com/twitter/server/lint/LoggingRules.scala b/server/src/main/scala/com/twitter/server/lint/LoggingRules.scala index 7c3dd9bd..0ed662ef 100644 --- a/server/src/main/scala/com/twitter/server/lint/LoggingRules.scala +++ b/server/src/main/scala/com/twitter/server/lint/LoggingRules.scala @@ -15,9 +15,7 @@ object LoggingRules { private[lint] def issues(jarPaths: Seq[String]): Seq[Issue] = { if (jarPaths.length > 1) { - jarPaths.map { jarPath => - Issue(jarName(jarPath)) - } + jarPaths.map { jarPath => Issue(jarName(jarPath)) } } else { Nil } diff --git a/server/src/main/scala/com/twitter/server/util/MetricSchemaSource.scala b/server/src/main/scala/com/twitter/server/util/MetricSchemaSource.scala index 864ddb40..f278fb0d 100644 --- a/server/src/main/scala/com/twitter/server/util/MetricSchemaSource.scala +++ b/server/src/main/scala/com/twitter/server/util/MetricSchemaSource.scala @@ -32,9 +32,7 @@ private[server] class MetricSchemaSource( /** Returns all schemas */ def schemaList(): Iterable[MetricSchema] = synchronized { registry - .foldLeft(IndexedSeq[MetricSchema]()) { (seq, r) => - seq ++ r.schemas().values - } + .foldLeft(IndexedSeq[MetricSchema]()) { (seq, r) => seq ++ r.schemas().values } } /** Returns true if the map contains `key` and false otherwise. */ @@ -45,8 +43,6 @@ private[server] class MetricSchemaSource( /** Returns the set of stat keys. */ def keySet: Set[String] = synchronized { registry - .foldLeft(Set[String]()) { (set, r) => - set ++ r.schemas().keySet - } + .foldLeft(Set[String]()) { (set, r) => set ++ r.schemas().keySet } } } diff --git a/server/src/main/scala/com/twitter/server/util/MetricSource.scala b/server/src/main/scala/com/twitter/server/util/MetricSource.scala index 299a2e99..8f5ff191 100644 --- a/server/src/main/scala/com/twitter/server/util/MetricSource.scala +++ b/server/src/main/scala/com/twitter/server/util/MetricSource.scala @@ -17,18 +17,14 @@ private[server] object MetricSource { * which allows for stale StatEntries up to `refreshInterval`. */ private[server] class MetricSource( - registry: () => Seq[StatsRegistry] = { () => - MetricSource.registry - }, + registry: () => Seq[StatsRegistry] = { () => MetricSource.registry }, refreshInterval: Duration = 1.second) { private[this] var lastRefresh = Time.now - refreshInterval private[this] var underlying: Map[String, StatEntry] = Map.empty private[this] def refresh(): Unit = { if (Time.now - lastRefresh > refreshInterval) { - val newStats = registry().foldLeft(Map[String, StatEntry]()) { (map, r) => - map ++ r() - } + val newStats = registry().foldLeft(Map[String, StatEntry]()) { (map, r) => map ++ r() } underlying = newStats lastRefresh = Time.now } diff --git a/server/src/main/scala/com/twitter/server/view/ThreadsView.scala b/server/src/main/scala/com/twitter/server/view/ThreadsView.scala index bf256be8..d14ecc3a 100644 --- a/server/src/main/scala/com/twitter/server/view/ThreadsView.scala +++ b/server/src/main/scala/com/twitter/server/view/ThreadsView.scala @@ -58,9 +58,7 @@ private object ThreadsView { */ private[server] class ThreadsView(all: Seq[ThreadInfo], deadlockedIds: Seq[Long]) { private def summary: String = { - val filtered = all.filter { info => - deadlockedIds.contains(info.thread.getId) - } + val filtered = all.filter { info => deadlockedIds.contains(info.thread.getId) } val deadlockLinks = if (filtered.isEmpty) "none" diff --git a/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala b/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala index 236d1546..e8871553 100644 --- a/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala +++ b/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala @@ -176,9 +176,7 @@ class AdminHttpServerTest extends FunSuite with Eventually with IntegrationPatie override def main(): Unit = { val p = new Promise[Unit] var sawClose = false - val drainingClosable = Closable.make { _ => - sawClose = true; p - } + val drainingClosable = Closable.make { _ => sawClose = true; p } closeOnExit(drainingClosable) val closeF: Future[Unit] = close(Time.now + 10.seconds) assert(sawClose) @@ -208,9 +206,7 @@ class AdminHttpServerTest extends FunSuite with Eventually with IntegrationPatie override protected def exitOnError(throwable: Throwable): Unit = () override def main(): Unit = { - val drainingClosable = Closable.make { _ => - Future.never - } + val drainingClosable = Closable.make { _ => Future.never } closeOnExit(drainingClosable) val closeF: Future[Unit] = close(Time.now + 10.seconds) // `drainingClosable` keeps the admin server from shutting down diff --git a/server/src/test/scala/com/twitter/server/handler/MetricTypeQueryHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/MetricTypeQueryHandlerTest.scala index 999fd4d4..5b16df90 100644 --- a/server/src/test/scala/com/twitter/server/handler/MetricTypeQueryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/MetricTypeQueryHandlerTest.scala @@ -50,17 +50,13 @@ class MetricTypeQueryHandlerTest extends FunSuite { override def apply(): Map[String, StatEntry] = stats.toMap } - val latchedMetricSource = new MetricSource( - () => { - Seq(latchedStatsRegistry) - } - ) + val latchedMetricSource = new MetricSource(() => { + Seq(latchedStatsRegistry) + }) - val unlatchedMetricSource = new MetricSource( - () => { - Seq(unlatchedStatsRegistry) - } - ) + val unlatchedMetricSource = new MetricSource(() => { + Seq(unlatchedStatsRegistry) + }) // generate artificial WithHistogramDetails as per HistogramQueryHandlerTest. val histos: InMemoryStatsReceiver = { diff --git a/server/src/test/scala/com/twitter/server/handler/ShutdownHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/ShutdownHandlerTest.scala index dcb081e4..7fd0ac04 100644 --- a/server/src/test/scala/com/twitter/server/handler/ShutdownHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/ShutdownHandlerTest.scala @@ -51,9 +51,7 @@ class ShutdownHandlerTest extends FunSuite { } test("fail when an invalid grace parameter is specified") { - val closer = Closer.mk { _ => - fail() - } + val closer = Closer.mk { _ => fail() } val handler = new ShutdownHandler(closer) val rsp = await(handler(Request(Method.Post, "/foo?grace=5"))) assert(rsp.status == Status.BadRequest) @@ -62,9 +60,7 @@ class ShutdownHandlerTest extends FunSuite { test("do not close when given a GET request") { val now = Time.now - val closer = Closer.mk { deadline => - assert(deadline == now + 1.second) - } + val closer = Closer.mk { deadline => assert(deadline == now + 1.second) } val handler = new ShutdownHandler(closer) val rsp = await(handler(Request(Method.Get, "/foo"))) assert(rsp.status == Status.MethodNotAllowed) diff --git a/server/src/test/scala/com/twitter/server/handler/ToggleHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/ToggleHandlerTest.scala index d8f196e6..e1121ab8 100644 --- a/server/src/test/scala/com/twitter/server/handler/ToggleHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/ToggleHandlerTest.scala @@ -31,12 +31,8 @@ class ToggleHandlerTest extends FunSuite { assert(mappings.keySet == libs.libraries.map(_.libraryName).toSet) // check the empty ToggleMap - assert(libs.libraries.forall { lib => - lib.libraryName.startsWith("com.twitter.Empty") - }) - assert(libs.libraries.forall { lib => - lib.toggles.isEmpty - }) + assert(libs.libraries.forall { lib => lib.libraryName.startsWith("com.twitter.Empty") }) + assert(libs.libraries.forall { lib => lib.toggles.isEmpty }) } test("toLibraryToggles") { diff --git a/server/src/test/scala/com/twitter/server/util/HtmlUtilsTest.scala b/server/src/test/scala/com/twitter/server/util/HtmlUtilsTest.scala index 7bafc6bf..29941b48 100644 --- a/server/src/test/scala/com/twitter/server/util/HtmlUtilsTest.scala +++ b/server/src/test/scala/com/twitter/server/util/HtmlUtilsTest.scala @@ -17,9 +17,7 @@ class HtmlUtilsTest extends FunSuite with ScalaCheckDrivenPropertyChecks { } test("escapeHtml leaves Strings untouched if they do not need escaping") { - forAll(Gen.alphaStr) { s => - assert(s == HtmlUtils.escapeHtml(s)) - } + forAll(Gen.alphaStr) { s => assert(s == HtmlUtils.escapeHtml(s)) } } test("escapeHtml escapes Strings if they need escaping") { diff --git a/server/src/test/scala/com/twitter/server/util/MetricSourceTest.scala b/server/src/test/scala/com/twitter/server/util/MetricSourceTest.scala index 6c185923..e383ea38 100644 --- a/server/src/test/scala/com/twitter/server/util/MetricSourceTest.scala +++ b/server/src/test/scala/com/twitter/server/util/MetricSourceTest.scala @@ -13,9 +13,7 @@ private[server] object MetricSourceTest { def apply() = underlying; override val latched: Boolean = false } - val registry = { () => - Seq(sr) - } + val registry = { () => Seq(sr) } val source = new MetricSource(registry, 1.second) } } diff --git a/server/src/test/scala/com/twitter/server/view/StackRegistryViewTest.scala b/server/src/test/scala/com/twitter/server/view/StackRegistryViewTest.scala index be6c3610..a8dde6bb 100644 --- a/server/src/test/scala/com/twitter/server/view/StackRegistryViewTest.scala +++ b/server/src/test/scala/com/twitter/server/view/StackRegistryViewTest.scala @@ -28,9 +28,7 @@ private[server] object StackRegistryViewTest { new Stack.Module0[Int => Int] { val role = Stack.Role("bar") val description = "adds 2 to every value." - def make(next: Int => Int) = { i => - next(i + 2) - } + def make(next: Int => Int) = { i => next(i + 2) } } } From 3a1cd258c6432ba3f43eee1daf13e1137e9ef01f Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Wed, 1 Apr 2020 17:41:14 +0000 Subject: [PATCH 150/409] Twitter-oss: Prepare OSS libraries for release 20.4.0 Problem We want to release the next versions of our Twitter OSS libraries 20.4.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-9579 Differential Revision: https://phabricator.twitter.biz/D459190 --- CHANGELOG.rst | 5 +++++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e8791823..65f9bb09 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +20.4.0 +------ + +No Changes + 20.3.0 ------ diff --git a/README.md b/README.md index 5baf251e..7ff7d5c9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index f27d3dce..78c87e36 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.4.0-SNAPSHOT" +val releaseVersion = "20.4.0" val jacksonVersion = "2.9.9" val jacksonDatabindVersion = "2.9.10.1" From 6272653b792914c0eb99a5893e90f924e605d71a Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 1 Apr 2020 20:20:22 +0000 Subject: [PATCH 151/409] compiler_option_sets and tags are lists not sets This change is a no-op - nothing should be observably different. Some of our tools are more comfortable with lists than sets, so use them more broadly. This change was generated with: ``` rg -g BUILD -l compiler_option_sets | xargs perl -0 -pi -e 's#compiler_option_sets = {([^}]*)}#compiler_option_sets = [\1]#mg' rg -g BUILD -l tags | xargs perl -0 -pi -e 's#tags = {([^}]*)}#tags = [\1]#mg' ``` Differential Revision: https://phabricator.twitter.biz/D459529 --- logback-classic/src/main/scala/BUILD | 2 +- logback-classic/src/test/scala/BUILD | 2 +- opencensus/src/main/scala/com/twitter/server/opencensus/BUILD | 2 +- server/src/main/scala/BUILD | 2 +- server/src/test/java/BUILD | 2 +- server/src/test/scala/BUILD | 2 +- slf4j-jdk14/src/main/scala/BUILD | 2 +- slf4j-jdk14/src/test/scala/BUILD | 2 +- slf4j-log4j12/src/main/scala/BUILD | 2 +- slf4j-log4j12/src/test/scala/BUILD | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/logback-classic/src/main/scala/BUILD b/logback-classic/src/main/scala/BUILD index 9ebd702a..96032250 100644 --- a/logback-classic/src/main/scala/BUILD +++ b/logback-classic/src/main/scala/BUILD @@ -1,6 +1,6 @@ scala_library( sources = ["**/*.scala"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], provides = scala_artifact( org = "com.twitter", name = "twitter-server-logback-classic", diff --git a/logback-classic/src/test/scala/BUILD b/logback-classic/src/test/scala/BUILD index e5e098b5..e84ad4d1 100644 --- a/logback-classic/src/test/scala/BUILD +++ b/logback-classic/src/test/scala/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = ["**/*.scala"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/ch/qos/logback:logback-classic", "3rdparty/jvm/junit", diff --git a/opencensus/src/main/scala/com/twitter/server/opencensus/BUILD b/opencensus/src/main/scala/com/twitter/server/opencensus/BUILD index c8b18d35..7f118720 100644 --- a/opencensus/src/main/scala/com/twitter/server/opencensus/BUILD +++ b/opencensus/src/main/scala/com/twitter/server/opencensus/BUILD @@ -1,6 +1,6 @@ scala_library( sources = ["*.scala"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/io/opencensus:opencensus-api", "3rdparty/jvm/io/opencensus:opencensus-contrib-zpages", diff --git a/server/src/main/scala/BUILD b/server/src/main/scala/BUILD index 51ceecf7..86752d13 100644 --- a/server/src/main/scala/BUILD +++ b/server/src/main/scala/BUILD @@ -1,6 +1,6 @@ scala_library( sources = ["**/*.scala"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], provides = scala_artifact( org = "com.twitter", name = "twitter-server", diff --git a/server/src/test/java/BUILD b/server/src/test/java/BUILD index add1710d..500c6839 100644 --- a/server/src/test/java/BUILD +++ b/server/src/test/java/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = ["**/*.java"], - compiler_option_sets = {}, + compiler_option_sets = [], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", diff --git a/server/src/test/scala/BUILD b/server/src/test/scala/BUILD index 7e988c97..a4afebd9 100644 --- a/server/src/test/scala/BUILD +++ b/server/src/test/scala/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = ["**/*.scala"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", "3rdparty/jvm/com/fasterxml/jackson/module:jackson-module-scala", diff --git a/slf4j-jdk14/src/main/scala/BUILD b/slf4j-jdk14/src/main/scala/BUILD index 848076c3..ee7e5e59 100644 --- a/slf4j-jdk14/src/main/scala/BUILD +++ b/slf4j-jdk14/src/main/scala/BUILD @@ -1,6 +1,6 @@ scala_library( sources = ["**/*.scala"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], provides = scala_artifact( org = "com.twitter", name = "twitter-server-slf4j-jdk14", diff --git a/slf4j-jdk14/src/test/scala/BUILD b/slf4j-jdk14/src/test/scala/BUILD index eec50c3a..bc950e9c 100644 --- a/slf4j-jdk14/src/test/scala/BUILD +++ b/slf4j-jdk14/src/test/scala/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = ["**/*.scala"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", diff --git a/slf4j-log4j12/src/main/scala/BUILD b/slf4j-log4j12/src/main/scala/BUILD index 618400f2..618c2c2b 100644 --- a/slf4j-log4j12/src/main/scala/BUILD +++ b/slf4j-log4j12/src/main/scala/BUILD @@ -1,6 +1,6 @@ scala_library( sources = ["**/*.scala"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], provides = scala_artifact( org = "com.twitter", name = "twitter-server-slf4j-log4j12", diff --git a/slf4j-log4j12/src/test/scala/BUILD b/slf4j-log4j12/src/test/scala/BUILD index 650695c6..0c0de1bd 100644 --- a/slf4j-log4j12/src/test/scala/BUILD +++ b/slf4j-log4j12/src/test/scala/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = ["**/*.scala"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/log4j", From 3cbf76a79ffa5c2ad9b4d1814826bfda0787f610 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 2 Apr 2020 18:56:55 +0000 Subject: [PATCH 152/409] Remove some extra BUILD files from util-core Move these targets into the main util-core BUILD file, because these files aren't providing much value, and their targets can exist in the place they're already tightly coupled into. NO_USER_HOOK=1 Differential Revision: https://phabricator.twitter.biz/D460349 --- server/src/main/scala/BUILD | 4 ++-- server/src/test/java/BUILD | 2 +- server/src/test/scala/BUILD | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/main/scala/BUILD b/server/src/main/scala/BUILD index 86752d13..8bb6025c 100644 --- a/server/src/main/scala/BUILD +++ b/server/src/main/scala/BUILD @@ -24,7 +24,7 @@ scala_library( "finagle/finagle-tunable/src/main/scala", "twitter-server/server/src/main/resources", "util/util-app/src/main/scala", - "util/util-core/src/main/scala", + "util/util-core:scala", "util/util-jvm/src/main/scala", "util/util-lint/src/main/scala", "util/util-registry/src/main/scala", @@ -40,7 +40,7 @@ scala_library( "finagle/finagle-base-http/src/main/scala", "finagle/finagle-http/src/main/scala", "util/util-app/src/main/scala", - "util/util-core/src/main/scala", + "util/util-core:scala", "util/util-lint/src/main/scala", "util/util-slf4j-api/src/main/scala", ], diff --git a/server/src/test/java/BUILD b/server/src/test/java/BUILD index 500c6839..2b95c9ca 100644 --- a/server/src/test/java/BUILD +++ b/server/src/test/java/BUILD @@ -12,7 +12,7 @@ junit_tests( "finagle/finagle-http/src/main/scala", "twitter-server/server/src/main/scala", "util/util-app/src/main/scala", - "util/util-core/src/main/scala", + "util/util-core:scala", "util/util-lint/src/main/scala", "util/util-slf4j-api/src/main/scala", "util/util-stats/src/main/scala", diff --git a/server/src/test/scala/BUILD b/server/src/test/scala/BUILD index a4afebd9..9fdf58a3 100644 --- a/server/src/test/scala/BUILD +++ b/server/src/test/scala/BUILD @@ -16,7 +16,7 @@ junit_tests( "twitter-server/server/src/main/scala", "twitter-server/server/src/test/resources", "util/util-app/src/main/scala", - "util/util-core/src/main/scala", + "util/util-core:scala", "util/util-lint/src/main/scala", "util/util-registry/src/main/scala", "util/util-slf4j-api/src/main/scala", From b049696d2963afe2b16ccd7974c4ff31fa657ed0 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Thu, 2 Apr 2020 23:15:32 +0000 Subject: [PATCH 153/409] Twitter-oss: update OSS libraries post-release to 20.5.0 Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 20.5.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D460793 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7ff7d5c9..5baf251e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 78c87e36..78b58516 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.4.0" +val releaseVersion = "20.5.0-SNAPSHOT" val jacksonVersion = "2.9.9" val jacksonDatabindVersion = "2.9.10.1" From cafbd3a8185531a210d6d0dffa99faa58d338372 Mon Sep 17 00:00:00 2001 From: Ian Bennett Date: Tue, 7 Apr 2020 19:01:01 +0000 Subject: [PATCH 154/409] python 2 to 3 updates Problem/Solution We have many resources that are Python 2 based, which is no longer supported. We need to move them to Python 3. JIRA Issues: CSL-9429 Differential Revision: https://phabricator.twitter.biz/D437372 --- doc/src/sphinx/conf.py | 16 ++++++++-------- doc/src/sphinx/exts/includecode.py | 6 +++--- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/doc/src/sphinx/conf.py b/doc/src/sphinx/conf.py index 081ee96a..ce736f5b 100644 --- a/doc/src/sphinx/conf.py +++ b/doc/src/sphinx/conf.py @@ -32,8 +32,8 @@ html_show_sphinx = False -project = u'TwitterServer' -copyright = u'{} Twitter, Inc'.format(datetime.datetime.now().year) +project = 'TwitterServer' +copyright = '{} Twitter, Inc'.format(datetime.datetime.now().year) version = '' release = '' htmlhelp_basename = "twitter-server" @@ -55,12 +55,12 @@ try: __import__('flask_theme_support') except ImportError as e: - print '-' * 74 - print 'Warning: Flask themes unavailable. Building with default theme' - print 'If you want the Flask themes, run this command and build again:' - print - print ' git submodule update --init' - print '-' * 74 + print('-' * 74) + print('Warning: Flask themes unavailable. Building with default theme') + print('If you want the Flask themes, run this command and build again:') + print() + print(' git submodule update --init') + print('-' * 74) pygments_style = 'tango' html_theme = 'default' diff --git a/doc/src/sphinx/exts/includecode.py b/doc/src/sphinx/exts/includecode.py index c058c7c8..47d08b6a 100644 --- a/doc/src/sphinx/exts/includecode.py +++ b/doc/src/sphinx/exts/includecode.py @@ -108,12 +108,12 @@ def countwhile(predicate, iterable): else: return count - nonempty = filter(lambda l: l.strip(), lines) - tabcounts = map(lambda l: countwhile(lambda c: c == ' ', l), nonempty) + nonempty = [l for l in lines if l.strip()] + tabcounts = [countwhile(lambda c: c == ' ', l) for l in nonempty] tabshift = min(tabcounts) if tabcounts else 0 if tabshift > 0: - lines = map(lambda l: l[tabshift:] if len(l) > tabshift else l, lines) + lines = [l[tabshift:] if len(l) > tabshift else l for l in lines] prepend = self.options.get('prepend') append = self.options.get('append') From e8b429bb20f20061e59c90d90d63a9d0b21a77cf Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 8 Apr 2020 21:11:44 +0000 Subject: [PATCH 155/409] Inline alias target JIRA Issues: DPB-12260 NO_USER_HOOK=1 Differential Revision: https://phabricator.twitter.biz/D463744 --- logback-classic/src/test/scala/BUILD | 2 +- server/src/main/scala/BUILD | 2 +- server/src/test/java/BUILD | 2 +- server/src/test/scala/BUILD | 2 +- slf4j-jdk14/src/test/scala/BUILD | 2 +- slf4j-log4j12/src/test/scala/BUILD | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/logback-classic/src/test/scala/BUILD b/logback-classic/src/test/scala/BUILD index e84ad4d1..c46ecae9 100644 --- a/logback-classic/src/test/scala/BUILD +++ b/logback-classic/src/test/scala/BUILD @@ -6,7 +6,7 @@ junit_tests( "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/slf4j:slf4j-api", - "finagle/finagle-core/src/main/scala", + "finagle/finagle-core/src/main", "twitter-server/logback-classic/src/main/scala", "twitter-server/server/src/main/scala", "util/util-slf4j-api/src/main/scala", diff --git a/server/src/main/scala/BUILD b/server/src/main/scala/BUILD index 8bb6025c..0251e3de 100644 --- a/server/src/main/scala/BUILD +++ b/server/src/main/scala/BUILD @@ -15,7 +15,7 @@ scala_library( "3rdparty/jvm/io/netty:netty4", "3rdparty/jvm/org/slf4j:slf4j-api", "finagle/finagle-base-http/src/main/scala", - "finagle/finagle-core/src/main/scala", + "finagle/finagle-core/src/main", "finagle/finagle-http/src/main/scala", "finagle/finagle-netty4-http/src/main/scala", "finagle/finagle-netty4/src/main/scala", diff --git a/server/src/test/java/BUILD b/server/src/test/java/BUILD index 2b95c9ca..7da6c4d1 100644 --- a/server/src/test/java/BUILD +++ b/server/src/test/java/BUILD @@ -8,7 +8,7 @@ junit_tests( "3rdparty/jvm/org/scalacheck", "3rdparty/jvm/org/slf4j:slf4j-api", "3rdparty/jvm/org/slf4j:slf4j-nop", - "finagle/finagle-core/src/main/scala", + "finagle/finagle-core/src/main", "finagle/finagle-http/src/main/scala", "twitter-server/server/src/main/scala", "util/util-app/src/main/scala", diff --git a/server/src/test/scala/BUILD b/server/src/test/scala/BUILD index 9fdf58a3..cea41817 100644 --- a/server/src/test/scala/BUILD +++ b/server/src/test/scala/BUILD @@ -10,7 +10,7 @@ junit_tests( "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/slf4j:slf4j-api", "3rdparty/jvm/org/slf4j:slf4j-nop", - "finagle/finagle-core/src/main/scala", + "finagle/finagle-core/src/main", "finagle/finagle-http/src/main/scala", "finagle/finagle-toggle", "twitter-server/server/src/main/scala", diff --git a/slf4j-jdk14/src/test/scala/BUILD b/slf4j-jdk14/src/test/scala/BUILD index bc950e9c..ff32cd00 100644 --- a/slf4j-jdk14/src/test/scala/BUILD +++ b/slf4j-jdk14/src/test/scala/BUILD @@ -7,7 +7,7 @@ junit_tests( "3rdparty/jvm/org/scalacheck", "3rdparty/jvm/org/scalatest", "finagle/finagle-base-http/src/main/scala", - "finagle/finagle-core/src/main/scala", + "finagle/finagle-core/src/main", "finagle/finagle-http/src/main/scala", "twitter-server/server/src/main/scala", "twitter-server/slf4j-jdk14/src/main/scala", diff --git a/slf4j-log4j12/src/test/scala/BUILD b/slf4j-log4j12/src/test/scala/BUILD index 0c0de1bd..25405f48 100644 --- a/slf4j-log4j12/src/test/scala/BUILD +++ b/slf4j-log4j12/src/test/scala/BUILD @@ -6,7 +6,7 @@ junit_tests( "3rdparty/jvm/log4j", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/slf4j:slf4j-api", - "finagle/finagle-core/src/main/scala", + "finagle/finagle-core/src/main", "twitter-server/server/src/main/scala", "twitter-server/slf4j-log4j12/src/main/scala", "util/util-slf4j-api/src/main/scala", From fec56de1df3b821e9f0d64861c2dcd49043cbc48 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 8 Apr 2020 22:15:14 +0000 Subject: [PATCH 156/409] Inline alias target JIRA Issues: DPB-12260 NO_USER_HOOK=1 Differential Revision: https://phabricator.twitter.biz/D463863 --- logback-classic/src/main/scala/BUILD | 2 +- server/src/main/scala/BUILD | 4 ++-- slf4j-jdk14/src/main/scala/BUILD | 2 +- slf4j-jdk14/src/test/scala/BUILD | 2 +- slf4j-log4j12/src/main/scala/BUILD | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/logback-classic/src/main/scala/BUILD b/logback-classic/src/main/scala/BUILD index 96032250..d706bba7 100644 --- a/logback-classic/src/main/scala/BUILD +++ b/logback-classic/src/main/scala/BUILD @@ -15,7 +15,7 @@ scala_library( "3rdparty/jvm/org/slf4j:jul-to-slf4j", "3rdparty/jvm/org/slf4j:log4j-over-slf4j", "3rdparty/jvm/org/slf4j:slf4j-api", - "finagle/finagle-base-http/src/main/scala", + "finagle/finagle-base-http/src/main", "finagle/finagle-http/src/main/scala", "twitter-server/logback-classic/src/main/resources", "twitter-server/server/src/main/scala", diff --git a/server/src/main/scala/BUILD b/server/src/main/scala/BUILD index 0251e3de..88344c99 100644 --- a/server/src/main/scala/BUILD +++ b/server/src/main/scala/BUILD @@ -14,7 +14,7 @@ scala_library( "3rdparty/jvm/com/fasterxml/jackson/module:jackson-module-scala", "3rdparty/jvm/io/netty:netty4", "3rdparty/jvm/org/slf4j:slf4j-api", - "finagle/finagle-base-http/src/main/scala", + "finagle/finagle-base-http/src/main", "finagle/finagle-core/src/main", "finagle/finagle-http/src/main/scala", "finagle/finagle-netty4-http/src/main/scala", @@ -37,7 +37,7 @@ scala_library( "3rdparty/jvm/com/fasterxml/jackson/core:jackson-annotations", "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", "3rdparty/jvm/org/slf4j:slf4j-api", - "finagle/finagle-base-http/src/main/scala", + "finagle/finagle-base-http/src/main", "finagle/finagle-http/src/main/scala", "util/util-app/src/main/scala", "util/util-core:scala", diff --git a/slf4j-jdk14/src/main/scala/BUILD b/slf4j-jdk14/src/main/scala/BUILD index ee7e5e59..86c83f9e 100644 --- a/slf4j-jdk14/src/main/scala/BUILD +++ b/slf4j-jdk14/src/main/scala/BUILD @@ -13,7 +13,7 @@ scala_library( "3rdparty/jvm/org/slf4j:log4j-over-slf4j", "3rdparty/jvm/org/slf4j:slf4j-api", "3rdparty/jvm/org/slf4j:slf4j-jdk14", - "finagle/finagle-base-http/src/main/scala", + "finagle/finagle-base-http/src/main", "finagle/finagle-http/src/main/scala", "twitter-server/server/src/main/scala", "twitter-server/slf4j-jdk14/src/main/resources", diff --git a/slf4j-jdk14/src/test/scala/BUILD b/slf4j-jdk14/src/test/scala/BUILD index ff32cd00..68132d99 100644 --- a/slf4j-jdk14/src/test/scala/BUILD +++ b/slf4j-jdk14/src/test/scala/BUILD @@ -6,7 +6,7 @@ junit_tests( "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scalacheck", "3rdparty/jvm/org/scalatest", - "finagle/finagle-base-http/src/main/scala", + "finagle/finagle-base-http/src/main", "finagle/finagle-core/src/main", "finagle/finagle-http/src/main/scala", "twitter-server/server/src/main/scala", diff --git a/slf4j-log4j12/src/main/scala/BUILD b/slf4j-log4j12/src/main/scala/BUILD index 618c2c2b..1de45115 100644 --- a/slf4j-log4j12/src/main/scala/BUILD +++ b/slf4j-log4j12/src/main/scala/BUILD @@ -14,7 +14,7 @@ scala_library( "3rdparty/jvm/org/slf4j:jul-to-slf4j", "3rdparty/jvm/org/slf4j:slf4j-api", "3rdparty/jvm/org/slf4j:slf4j-log4j12", - "finagle/finagle-base-http/src/main/scala", + "finagle/finagle-base-http/src/main", "finagle/finagle-http/src/main/scala", "twitter-server/server/src/main/scala", "twitter-server/slf4j-log4j12/src/main/resources", From 6d3e8f0ce890c218e774336d3335e3db9b22ccf6 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 15 Apr 2020 19:12:07 +0000 Subject: [PATCH 157/409] Add missing deps and exports Problem Some code does not fully declare their dependencies, so things which depend on them need to put more on the classpath when compiling than necessary. Solution Add the missing dependencies. Result Smaller classpaths can be used when compiling dependees of this code. Differential Revision: https://phabricator.twitter.biz/D464275 --- server/src/main/scala/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/scala/BUILD b/server/src/main/scala/BUILD index 88344c99..70e2cb8c 100644 --- a/server/src/main/scala/BUILD +++ b/server/src/main/scala/BUILD @@ -13,6 +13,7 @@ scala_library( "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", "3rdparty/jvm/com/fasterxml/jackson/module:jackson-module-scala", "3rdparty/jvm/io/netty:netty4", + "3rdparty/jvm/io/netty:netty4-codec-http", "3rdparty/jvm/org/slf4j:slf4j-api", "finagle/finagle-base-http/src/main", "finagle/finagle-core/src/main", From 036644585d78a1c009f5303e01dea476c27dbd7e Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Thu, 23 Apr 2020 23:56:11 +0000 Subject: [PATCH 158/409] Twitter-oss: Prepare OSS libraries for release 20.4.1 Problem We want to release the next versions of our Twitter OSS libraries 20.4.1 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D473181 --- CHANGELOG.rst | 5 +++++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 65f9bb09..3473e9f7 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +20.4.1 +------ + +No Changes + 20.4.0 ------ diff --git a/README.md b/README.md index 5baf251e..7ff7d5c9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 78b58516..12e8e688 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.5.0-SNAPSHOT" +val releaseVersion = "20.4.1" val jacksonVersion = "2.9.9" val jacksonDatabindVersion = "2.9.10.1" From 2ebc5f30148d2c2b4fc3ebb23ab960551ac9f298 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Mon, 27 Apr 2020 16:44:33 +0000 Subject: [PATCH 159/409] Twitter-oss: update OSS libraries post-release to 20.5.0 Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 20.5.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D474463 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7ff7d5c9..5baf251e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 12e8e688..78b58516 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.4.1" +val releaseVersion = "20.5.0-SNAPSHOT" val jacksonVersion = "2.9.9" val jacksonDatabindVersion = "2.9.10.1" From 2fc34c6abc04b52e1e4da93aae8425ba406bccfc Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Wed, 29 Apr 2020 05:14:39 +0000 Subject: [PATCH 160/409] 3rdparty: Bump jackson to 2.11.0 Problem / Solution / Result There's a new version of jackson, let's update to the new version. JIRA Issues: CSL-9562, CSL-9723 Differential Revision: https://phabricator.twitter.biz/D457496 --- CHANGELOG.rst | 5 +++++ build.sbt | 5 ++--- .../scala/com/twitter/server/HistogramQueryHandlerTest.scala | 2 +- .../test/scala/com/twitter/server/handler/JsonHelper.scala | 2 +- .../twitter/server/handler/MetricTypeQueryHandlerTest.scala | 3 +-- .../com/twitter/server/handler/RegistryHandlerTest.scala | 3 +-- .../com/twitter/server/handler/TunableHandlerTest.scala | 3 +-- .../com/twitter/server/util/MetricSchemaJsonModuleTest.scala | 3 +-- 8 files changed, 13 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3473e9f7..f1e966f0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Runtime Behavior Changes +~~~~~~~~~~~~~~~~~~~~~~~~ + +* Bump jackson version to 2.11.0. ``PHAB_ID=D457496`` + 20.4.1 ------ diff --git a/build.sbt b/build.sbt index 78b58516..5cf0860a 100644 --- a/build.sbt +++ b/build.sbt @@ -3,11 +3,10 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch val releaseVersion = "20.5.0-SNAPSHOT" -val jacksonVersion = "2.9.9" -val jacksonDatabindVersion = "2.9.10.1" +val jacksonVersion = "2.11.0" val jacksonLibs = Seq( "com.fasterxml.jackson.core" % "jackson-core" % jacksonVersion, - "com.fasterxml.jackson.core" % "jackson-databind" % jacksonDatabindVersion, + "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion, "com.fasterxml.jackson.module" %% "jackson-module-scala" % jacksonVersion exclude ("com.google.guava", "guava") ) val opencensusVersion = "0.19.1" diff --git a/server/src/test/scala/com/twitter/server/HistogramQueryHandlerTest.scala b/server/src/test/scala/com/twitter/server/HistogramQueryHandlerTest.scala index 33278d11..b8531894 100644 --- a/server/src/test/scala/com/twitter/server/HistogramQueryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/HistogramQueryHandlerTest.scala @@ -2,7 +2,7 @@ package com.twitter.server import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.scala.DefaultScalaModule -import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper +import com.fasterxml.jackson.module.scala.ScalaObjectMapper import com.twitter.conversions.DurationOps._ import com.twitter.finagle.http.{Request, Response} import com.twitter.finagle.stats.{BucketAndCount, InMemoryStatsReceiver} diff --git a/server/src/test/scala/com/twitter/server/handler/JsonHelper.scala b/server/src/test/scala/com/twitter/server/handler/JsonHelper.scala index 88c2339c..59030a37 100644 --- a/server/src/test/scala/com/twitter/server/handler/JsonHelper.scala +++ b/server/src/test/scala/com/twitter/server/handler/JsonHelper.scala @@ -1,7 +1,7 @@ package com.twitter.server.handler import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper +import com.fasterxml.jackson.module.scala.ScalaObjectMapper object JsonHelper { diff --git a/server/src/test/scala/com/twitter/server/handler/MetricTypeQueryHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/MetricTypeQueryHandlerTest.scala index 5b16df90..21199593 100644 --- a/server/src/test/scala/com/twitter/server/handler/MetricTypeQueryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/MetricTypeQueryHandlerTest.scala @@ -7,8 +7,7 @@ import com.twitter.util.{Await, Duration} import java.util.concurrent.TimeUnit import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.module.scala.DefaultScalaModule -import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper +import com.fasterxml.jackson.module.scala.{DefaultScalaModule, ScalaObjectMapper} import org.scalatest.FunSuite import scala.collection.mutable diff --git a/server/src/test/scala/com/twitter/server/handler/RegistryHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/RegistryHandlerTest.scala index 497c687d..a8cd84b0 100644 --- a/server/src/test/scala/com/twitter/server/handler/RegistryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/RegistryHandlerTest.scala @@ -1,8 +1,7 @@ package com.twitter.server.handler import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.module.scala.DefaultScalaModule -import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper +import com.fasterxml.jackson.module.scala.{DefaultScalaModule, ScalaObjectMapper} import com.twitter.util.registry.{GlobalRegistry, SimpleRegistry} import org.scalatest.FunSuite diff --git a/server/src/test/scala/com/twitter/server/handler/TunableHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/TunableHandlerTest.scala index b0db48da..9c1985e6 100644 --- a/server/src/test/scala/com/twitter/server/handler/TunableHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/TunableHandlerTest.scala @@ -1,8 +1,7 @@ package com.twitter.server.handler import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.module.scala.DefaultScalaModule -import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper +import com.fasterxml.jackson.module.scala.{DefaultScalaModule, ScalaObjectMapper} import com.twitter.conversions.DurationOps._ import com.twitter.finagle.http.{MediaType, Method, Request, Status} import com.twitter.util.Await diff --git a/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala b/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala index aaad8f0e..7de7373c 100644 --- a/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala +++ b/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala @@ -1,8 +1,7 @@ package com.twitter.server.util import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.module.scala.DefaultScalaModule -import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper +import com.fasterxml.jackson.module.scala.{DefaultScalaModule, ScalaObjectMapper} import com.twitter.finagle.stats._ import org.scalatest.FunSuite From efcb99c996fe98ab66755ac1982d73eb2bde2d56 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Fri, 1 May 2020 19:14:22 +0000 Subject: [PATCH 161/409] csl: Bump sbt version Problem/Solution Bump the sbt to 1.3.10 to help OSS experiment cross-building our libs with Dotty/Scala3. sbt-dotty requires 1.3.10. JIRA Issues: CSL-9718 Differential Revision: https://phabricator.twitter.biz/D476543 --- project/build.properties | 2 +- sbt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/project/build.properties b/project/build.properties index c0bab049..797e7ccf 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.2.8 +sbt.version=1.3.10 diff --git a/sbt b/sbt index e7f538ee..4cb7d05e 100755 --- a/sbt +++ b/sbt @@ -1,8 +1,8 @@ #!/bin/bash -sbtver=1.2.8 +sbtver=1.3.10 sbtjar=sbt-launch.jar -sbtsha128=073c169c6e1a47b8ae78a7a718b907424dedab30 +sbtsha128=f49861befe36746728182ea7efb30096c724b0d7 sbtrepo="https://repo1.maven.org/maven2/org/scala-sbt/sbt-launch" From bc89dba699ae6e8d9b398889720b7570a433cc58 Mon Sep 17 00:00:00 2001 From: Shane Delmore Date: Sat, 9 May 2020 21:24:22 +0000 Subject: [PATCH 162/409] Update scalafmt version Differential Revision: https://phabricator.twitter.biz/D481559 --- doc/src/sphinx/code/AdvancedServer.scala | 2 +- doc/src/sphinx/code/BasicServer.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/src/sphinx/code/AdvancedServer.scala b/doc/src/sphinx/code/AdvancedServer.scala index 54ac20ca..00b0e611 100644 --- a/doc/src/sphinx/code/AdvancedServer.scala +++ b/doc/src/sphinx/code/AdvancedServer.scala @@ -36,7 +36,7 @@ object AdvancedServer extends TwitterServer { } } - def main() { + def main(): Unit = { // We could create a new http server but in this case we use the // one already started for /admin/* endpoints. // The `TwitterServer` trait exposes an `adminHttpServer` that serve all routes diff --git a/doc/src/sphinx/code/BasicServer.scala b/doc/src/sphinx/code/BasicServer.scala index b3eeb999..acac65d8 100644 --- a/doc/src/sphinx/code/BasicServer.scala +++ b/doc/src/sphinx/code/BasicServer.scala @@ -15,7 +15,7 @@ object BasicServer extends TwitterServer { } } - def main() { + def main(): Unit = { val server = Http.serve(":8888", service) onExit { server.close() From d6c4db1fcf5b739ffd0cf5792a6c4feffa258aa0 Mon Sep 17 00:00:00 2001 From: Shane Delmore Date: Sun, 10 May 2020 01:37:41 +0000 Subject: [PATCH 163/409] reformat source with scalafmt 2.5.1 part 4 If you have merge conflicts see go/scalafmt NO_USER_HOOK=1 Differential Revision: https://phabricator.twitter.biz/D481624 --- .../scala/com/twitter/server/Lifecycle.scala | 3 ++- .../handler/ClientRegistryHandler.scala | 3 ++- .../handler/HistogramQueryHandler.scala | 24 ++++++++++++------- .../server/util/MetricSchemaSourceTest.scala | 3 ++- 4 files changed, 21 insertions(+), 12 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/Lifecycle.scala b/server/src/main/scala/com/twitter/server/Lifecycle.scala index b4325e9a..fc8268ec 100644 --- a/server/src/main/scala/com/twitter/server/Lifecycle.scala +++ b/server/src/main/scala/com/twitter/server/Lifecycle.scala @@ -46,7 +46,8 @@ object promoteBeforeServing object Lifecycle { private[server] class PromoteToOldGen( - runtimeArgs: Seq[String] = ManagementFactory.getRuntimeMXBean.getInputArguments.asScala.toList) { + runtimeArgs: Seq[String] = + ManagementFactory.getRuntimeMXBean.getInputArguments.asScala.toList) { private[this] val hasPromoted = new AtomicBoolean(false) private[server] def promoted: Boolean = hasPromoted.get() diff --git a/server/src/main/scala/com/twitter/server/handler/ClientRegistryHandler.scala b/server/src/main/scala/com/twitter/server/handler/ClientRegistryHandler.scala index 0144a708..cf353ade 100644 --- a/server/src/main/scala/com/twitter/server/handler/ClientRegistryHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/ClientRegistryHandler.scala @@ -43,7 +43,8 @@ private object ClientRegistryHandler { data-toggle="tooltip" data-placement="top" class="conn-trouble btn-xs btn-default"> ${escapeHtml(unavailable.toString)} unavailable endpoint(s)""" + aria-hidden="true"> ${escapeHtml( + unavailable.toString)} unavailable endpoint(s)""" }}
    diff --git a/server/src/main/scala/com/twitter/server/handler/HistogramQueryHandler.scala b/server/src/main/scala/com/twitter/server/handler/HistogramQueryHandler.scala index b0d0eeaf..bf87e6e5 100644 --- a/server/src/main/scala/com/twitter/server/handler/HistogramQueryHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/HistogramQueryHandler.scala @@ -358,19 +358,25 @@ private[server] class HistogramQueryHandler(details: WithHistogramDetails) htmlResponse(query) case Some("raw") => - jsonResponse(query, { counts: Seq[BucketAndCount] => - deliverData(Map(query -> counts), identity) - }) + jsonResponse( + query, + { counts: Seq[BucketAndCount] => + deliverData(Map(query -> counts), identity) + }) case Some("pdf") => - jsonResponse(query, { counts: Seq[BucketAndCount] => - deliverData(Map(query -> counts), x => pdf(x)) - }) + jsonResponse( + query, + { counts: Seq[BucketAndCount] => + deliverData(Map(query -> counts), x => pdf(x)) + }) case Some("cdf") => - jsonResponse(query, { counts: Seq[BucketAndCount] => - deliverData(Map(query -> counts), x => cdf(x)) - }) + jsonResponse( + query, + { counts: Seq[BucketAndCount] => + deliverData(Map(query -> counts), x => cdf(x)) + }) case _ => newResponse( diff --git a/server/src/test/scala/com/twitter/server/util/MetricSchemaSourceTest.scala b/server/src/test/scala/com/twitter/server/util/MetricSchemaSourceTest.scala index b4b856ce..27efbe79 100644 --- a/server/src/test/scala/com/twitter/server/util/MetricSchemaSourceTest.scala +++ b/server/src/test/scala/com/twitter/server/util/MetricSchemaSourceTest.scala @@ -79,7 +79,8 @@ class MetricSchemaSourceTest extends FunSuite { assert(emptyMetricSchemaSource.schemaList() == Seq()) } - test("contains accurately reflect the presence or absence of a Metric from the MetricSchema map") { + test( + "contains accurately reflect the presence or absence of a Metric from the MetricSchema map") { assert(metricSchemaSource.contains("my/cool/counter")) assert(!metricSchemaSource.contains("my/dull/counter")) assert(metricSchemaSource.contains("my/only/histo")) From 586d7fb772457adea9dac42d78c8ccfaa62bfa14 Mon Sep 17 00:00:00 2001 From: Eitan Adler Date: Mon, 18 May 2020 19:59:58 +0000 Subject: [PATCH 164/409] [logging] remove alias target for slf4j-log4j12 Problem / Solution Alias targets hide the true location of code and make it harder to follow go/111. remove all remaining uses of the alias, then remove target. JIRA Issues: REVSRVG-23220 Differential Revision: https://phabricator.twitter.biz/D485824 --- slf4j-log4j12/BUILD | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 slf4j-log4j12/BUILD diff --git a/slf4j-log4j12/BUILD b/slf4j-log4j12/BUILD deleted file mode 100644 index bfa2f4d4..00000000 --- a/slf4j-log4j12/BUILD +++ /dev/null @@ -1,5 +0,0 @@ -target( - dependencies = [ - "twitter-server/slf4j-log4j12/src/main/scala", - ], -) From 1ffd0f6b62a95c46b2d095487db0cc4401a0d0ba Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Mon, 18 May 2020 20:43:32 +0000 Subject: [PATCH 165/409] twitter-server: Better admin logging handler support Problem The logic to install a TwitterServer Admin logging handler for dynamic configuration of log levels per logging implementation suffers from a lack of specifity making it hard to detect when the incorrect case of multiple handlers being present on the classpath and thus loaded by `LoadService` arises. Solution Separate out loading of the logging handler to better detect the case of multiple handlers. Add a lint rule which is in violation when multiple are loaded. Make sure the TwitterServer admin only attempts to install a handler when there is only a single one specified. JIRA Issues: CSL-9764 Differential Revision: https://phabricator.twitter.biz/D484965 --- CHANGELOG.rst | 5 ++ ...com.twitter.server.handler.LoggingHandler} | 0 .../logback/classic/LoggingHandler.scala | 8 ++- .../com/twitter/server/AdminHttpServer.scala | 63 +++++++++++-------- .../server/handler/LoggingHandler.scala | 8 +++ .../server/handler/NoLoggingHandler.scala | 8 ++- .../twitter/server/lint/LoggingRules.scala | 10 +++ ...com.twitter.server.handler.LoggingHandler} | 0 .../handler/slf4j/jdk14/LoggingHandler.scala | 6 +- ...com.twitter.server.handler.LoggingHandler} | 0 .../slf4j/log4j12/LoggingHandler.scala | 6 +- 11 files changed, 77 insertions(+), 37 deletions(-) rename logback-classic/src/main/resources/META-INF/services/{com.twitter.server.handler.AdminHttpMuxHandler => com.twitter.server.handler.LoggingHandler} (100%) create mode 100644 server/src/main/scala/com/twitter/server/handler/LoggingHandler.scala rename slf4j-jdk14/src/main/resources/META-INF/services/{com.twitter.server.handler.AdminHttpMuxHandler => com.twitter.server.handler.LoggingHandler} (100%) rename slf4j-log4j12/src/main/resources/META-INF/services/{com.twitter.server.handler.AdminHttpMuxHandler => com.twitter.server.handler.LoggingHandler} (100%) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f1e966f0..cc4d9d2a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +* Make lookup of Admin `LoggingHandler` more resilient when multiple implementations are detected. + Now instead of perhaps using an incorrect handler the server will instead emit a lint rule violation + and not attempt to install a logging handler ensuring that only when a single `LoggingHandler` + is located that the functionality is enabled. ``PHAB_ID=D484965`` + Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/logback-classic/src/main/resources/META-INF/services/com.twitter.server.handler.AdminHttpMuxHandler b/logback-classic/src/main/resources/META-INF/services/com.twitter.server.handler.LoggingHandler similarity index 100% rename from logback-classic/src/main/resources/META-INF/services/com.twitter.server.handler.AdminHttpMuxHandler rename to logback-classic/src/main/resources/META-INF/services/com.twitter.server.handler.LoggingHandler diff --git a/logback-classic/src/main/scala/com/twitter/server/handler/logback/classic/LoggingHandler.scala b/logback-classic/src/main/scala/com/twitter/server/handler/logback/classic/LoggingHandler.scala index ac3fe070..8ff8b191 100644 --- a/logback-classic/src/main/scala/com/twitter/server/handler/logback/classic/LoggingHandler.scala +++ b/logback-classic/src/main/scala/com/twitter/server/handler/logback/classic/LoggingHandler.scala @@ -4,9 +4,8 @@ import ch.qos.logback.classic.{Level, Logger, LoggerContext} import com.twitter.finagle.http._ import com.twitter.io.Buf import com.twitter.server.Admin.Grouping -import com.twitter.server.handler.AdminHttpMuxHandler -import com.twitter.server.util.HttpUtils._ import com.twitter.server.util.HtmlUtils.escapeHtml +import com.twitter.server.util.HttpUtils._ import com.twitter.util.Future import com.twitter.util.logging.Logging import java.net.URLEncoder @@ -15,7 +14,10 @@ import org.slf4j.LoggerFactory import scala.annotation.tailrec import scala.collection.JavaConverters._ -private class LoggingHandler extends AdminHttpMuxHandler with Logging { +private class LoggingHandler extends com.twitter.server.handler.LoggingHandler with Logging { + + /** Implementation name */ + override val name: String = "logback-classic" implicit val loggerOrder: Ordering[Logger] = Ordering.by(_.getName) implicit val levelOrder: Ordering[Level] = Ordering.by(_.levelInt) diff --git a/server/src/main/scala/com/twitter/server/AdminHttpServer.scala b/server/src/main/scala/com/twitter/server/AdminHttpServer.scala index 449b8d5f..82c1e630 100644 --- a/server/src/main/scala/com/twitter/server/AdminHttpServer.scala +++ b/server/src/main/scala/com/twitter/server/AdminHttpServer.scala @@ -1,6 +1,6 @@ package com.twitter.server -import com.twitter.app.App +import com.twitter.app.{App, Flag} import com.twitter.finagle.client.ClientRegistry import com.twitter.finagle.filter.ServerAdmissionControl import com.twitter.finagle.http.{HttpMuxer, Method, Request, Response, Route => HttpRoute} @@ -11,7 +11,7 @@ import com.twitter.finagle.util.LoadService import com.twitter.finagle.{Http, ListeningServer, NullServer, Service} import com.twitter.server.Admin.Grouping import com.twitter.server.filters.AdminThreadPoolFilter -import com.twitter.server.handler.{AdminHttpMuxHandler, NoLoggingHandler} +import com.twitter.server.handler.{AdminHttpMuxHandler, LoggingHandler, NoLoggingHandler} import com.twitter.server.lint.LoggingRules import com.twitter.server.util.HttpUtils import com.twitter.server.view.{IndexView, NotFoundView} @@ -21,7 +21,6 @@ import com.twitter.util.{Future, Monitor, Time} import java.net.InetSocketAddress import org.slf4j.LoggerFactory import scala.language.reflectiveCalls -import com.twitter.app.Flag object AdminHttpServer { @@ -100,6 +99,22 @@ object AdminHttpServer { ): Route = { Route(path, handler, alias, group, includeInIndex, method) } + + /** Convert an AdminHttpMuxHandler to a AdminHttpServer.Route */ + private def muxHandlerToRoute(handler: AdminHttpMuxHandler): Route = { + AdminThreadPoolFilter.isolateRoute(Route.from(handler.route)) + } + + private val defaultLoggingHandlerRoute: Route = + AdminThreadPoolFilter.isolateRoute( + Route( + path = "/admin/logging", + handler = new NoLoggingHandler, + alias = "Logging", + group = Some(Grouping.Utilities), + includeInIndex = true + ) + ) } trait AdminHttpServer { self: App with Stats => @@ -132,12 +147,27 @@ trait AdminHttpServer { self: App with Stats => @volatile protected var adminHttpServer: ListeningServer = NullServer + // Look up a logging handler, will only be added if a single one is found. + private[this] val loggingHandlerRoute: Seq[Route] = { + val handlers = LoadService[LoggingHandler]() + if (handlers.length > 1) { + // add linting issue for multiple logging handlers + GlobalRules.get.add(LoggingRules.multipleLoggingHandlers(handlers.map(_.name))) + Seq(defaultLoggingHandlerRoute) + } else if (handlers.length == 1) { + // add the logging handler + handlers.map(muxHandlerToRoute) + } else { + // add linting issue for missing logging handler + GlobalRules.get.add(LoggingRules.NoLoggingHandler) + Seq(defaultLoggingHandlerRoute) + } + } + // We start with routes added via load service, note that these will be overridden // by any routes added in any call to updateMuxer(). private[this] val loadServiceRoutes: Seq[Route] = - LoadService[AdminHttpMuxHandler]() - .map(handler => Route.from(handler.route)) - .map(AdminThreadPoolFilter.isolateRoute) + LoadService[AdminHttpMuxHandler]().map(muxHandlerToRoute) ++ loggingHandlerRoute private[this] var allRoutes: Seq[Route] = loadServiceRoutes @@ -190,28 +220,7 @@ trait AdminHttpServer { self: App with Stats => */ protected def configureAdminHttpServer(server: Http.Server): Http.Server = server - /** Provide a "catch-all" LoggingHandler that displays a message about configuring a logging implementation */ - private[this] def addLoggingHandler(): Unit = { - if (!allRoutes.exists(_.path == "/admin/logging")) { - // add linting issue for un-configured logging handler - GlobalRules.get.add(LoggingRules.NoLoggingHandler) - - allRoutes = allRoutes ++ - Seq( - Route( - path = "/admin/logging", - handler = new NoLoggingHandler, - alias = "Logging", - group = Some(Grouping.Utilities), - includeInIndex = true - ) - ).map(AdminThreadPoolFilter.isolateRoute) - } - } - private[this] def updateMuxer(): Unit = { - addLoggingHandler() // ensure there is an /admin/logging handler - // create a service which multiplexes across all endpoints. val localMuxer = allRoutes.foldLeft(new HttpMuxer) { case (muxer, route) => diff --git a/server/src/main/scala/com/twitter/server/handler/LoggingHandler.scala b/server/src/main/scala/com/twitter/server/handler/LoggingHandler.scala new file mode 100644 index 00000000..9381d4f5 --- /dev/null +++ b/server/src/main/scala/com/twitter/server/handler/LoggingHandler.scala @@ -0,0 +1,8 @@ +package com.twitter.server.handler + +/** Marker trait for LoggingHandlers */ +trait LoggingHandler extends AdminHttpMuxHandler { + + /** Implementation name */ + def name: String +} diff --git a/server/src/main/scala/com/twitter/server/handler/NoLoggingHandler.scala b/server/src/main/scala/com/twitter/server/handler/NoLoggingHandler.scala index e28c2729..b6efebb5 100644 --- a/server/src/main/scala/com/twitter/server/handler/NoLoggingHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/NoLoggingHandler.scala @@ -9,10 +9,12 @@ import com.twitter.util.logging.Logger private[server] object NoLoggingHandler { val MissingLoggingImplMessageHeader: String = - "You have not configured a logging handler implementation for TwitterServer." + "You have not properly configured a logging handler implementation for TwitterServer." val MissingLoggingImplMessageBody: String = - "Please add a dependency on one of: twitter-server-logback-classic, " + - "twitter-server-slf4j-jdk14, or twitter-server-slf4j-log4j12." + "Please add a dependency on only one of: twitter-server-logback-classic, " + + "twitter-server-slf4j-jdk14, or twitter-server-slf4j-log4j12. Note: this page will appear " + + "when there is no dependency configured or when multiple conflicting dependencies have been " + + "detected. Please check the /admin/lint page for more information." } class NoLoggingHandler extends Service[Request, Response] { diff --git a/server/src/main/scala/com/twitter/server/lint/LoggingRules.scala b/server/src/main/scala/com/twitter/server/lint/LoggingRules.scala index 0ed662ef..01677136 100644 --- a/server/src/main/scala/com/twitter/server/lint/LoggingRules.scala +++ b/server/src/main/scala/com/twitter/server/lint/LoggingRules.scala @@ -48,4 +48,14 @@ object LoggingRules { ) { Seq(Issue("No logging handler implementation configured.")) } + + def multipleLoggingHandlers(names: Seq[String]): Rule = Rule( + Category.Configuration, + "Multiple Admin logging handler implementations detected", + "To properly configure the ability to dynamically change log levels, please specify only " + + "one of the supported TwitterServer logging implementations on your " + + "classpath: logback-classic, slf4j-log4j12, or slf4j-jdk14." + ) { + Seq(Issue(s"Multiple logging handler implementations found: ${names.mkString(", ")}")) + } } diff --git a/slf4j-jdk14/src/main/resources/META-INF/services/com.twitter.server.handler.AdminHttpMuxHandler b/slf4j-jdk14/src/main/resources/META-INF/services/com.twitter.server.handler.LoggingHandler similarity index 100% rename from slf4j-jdk14/src/main/resources/META-INF/services/com.twitter.server.handler.AdminHttpMuxHandler rename to slf4j-jdk14/src/main/resources/META-INF/services/com.twitter.server.handler.LoggingHandler diff --git a/slf4j-jdk14/src/main/scala/com/twitter/server/handler/slf4j/jdk14/LoggingHandler.scala b/slf4j-jdk14/src/main/scala/com/twitter/server/handler/slf4j/jdk14/LoggingHandler.scala index 9a5c5fc1..13ab82cb 100644 --- a/slf4j-jdk14/src/main/scala/com/twitter/server/handler/slf4j/jdk14/LoggingHandler.scala +++ b/slf4j-jdk14/src/main/scala/com/twitter/server/handler/slf4j/jdk14/LoggingHandler.scala @@ -4,7 +4,6 @@ import com.twitter.finagle.http._ import com.twitter.io.Buf import com.twitter.logging.{Level, Logger} import com.twitter.server.Admin.Grouping -import com.twitter.server.handler.AdminHttpMuxHandler import com.twitter.server.handler.slf4j.jdk14.LoggingHandler._ import com.twitter.server.util.HtmlUtils._ import com.twitter.server.util.HttpUtils._ @@ -83,7 +82,10 @@ private object LoggingHandler { * [[com.twitter.logging.Logger]] configuration state and allows for runtime changes * via HTTP query strings (?logger=&level=). */ -private class LoggingHandler extends AdminHttpMuxHandler { +private class LoggingHandler extends com.twitter.server.handler.LoggingHandler { + + /** Implementation name */ + override val name: String = "slf4j-jdk14" private[this] val levels = Logger.levels.values.toSeq.sorted(LoggingHandler.levelOrder) private[this] val logManager = java.util.logging.LogManager.getLogManager diff --git a/slf4j-log4j12/src/main/resources/META-INF/services/com.twitter.server.handler.AdminHttpMuxHandler b/slf4j-log4j12/src/main/resources/META-INF/services/com.twitter.server.handler.LoggingHandler similarity index 100% rename from slf4j-log4j12/src/main/resources/META-INF/services/com.twitter.server.handler.AdminHttpMuxHandler rename to slf4j-log4j12/src/main/resources/META-INF/services/com.twitter.server.handler.LoggingHandler diff --git a/slf4j-log4j12/src/main/scala/com/twitter/server/handler/slf4j/log4j12/LoggingHandler.scala b/slf4j-log4j12/src/main/scala/com/twitter/server/handler/slf4j/log4j12/LoggingHandler.scala index 311c9689..c6c1e790 100644 --- a/slf4j-log4j12/src/main/scala/com/twitter/server/handler/slf4j/log4j12/LoggingHandler.scala +++ b/slf4j-log4j12/src/main/scala/com/twitter/server/handler/slf4j/log4j12/LoggingHandler.scala @@ -3,7 +3,6 @@ package com.twitter.server.handler.slf4j.log4j12 import com.twitter.finagle.http._ import com.twitter.io.Buf import com.twitter.server.Admin.Grouping -import com.twitter.server.handler.AdminHttpMuxHandler import com.twitter.server.util.HtmlUtils._ import com.twitter.server.util.HttpUtils._ import com.twitter.util.Future @@ -12,7 +11,10 @@ import java.net.URLEncoder import org.apache.log4j.{Level, LogManager, Logger} import scala.collection.JavaConverters._ -private class LoggingHandler extends AdminHttpMuxHandler with Logging { +private class LoggingHandler extends com.twitter.server.handler.LoggingHandler with Logging { + + /** Implementation name */ + override def name: String = "slf4j-log412" implicit val loggerOrder: Ordering[Logger] = Ordering.by(_.getName) implicit val levelOrder: Ordering[Level] = Ordering.by(_.toInt) From ae717e3275af4a72846eec4e81a97e908b221205 Mon Sep 17 00:00:00 2001 From: Ian Bennett Date: Tue, 19 May 2020 00:27:27 +0000 Subject: [PATCH 166/409] Twitter-oss: Prepare OSS libraries for release 20.5.0 Problem We want to release the next versions of our Twitter OSS libraries 20.5.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-9788 Differential Revision: https://phabricator.twitter.biz/D486551 --- CHANGELOG.rst | 3 +++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index cc4d9d2a..4379ed2e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +20.5.0 +------ + * Make lookup of Admin `LoggingHandler` more resilient when multiple implementations are detected. Now instead of perhaps using an incorrect handler the server will instead emit a lint rule violation and not attempt to install a logging handler ensuring that only when a single `LoggingHandler` diff --git a/README.md b/README.md index 5baf251e..7ff7d5c9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 5cf0860a..90ac53e1 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.5.0-SNAPSHOT" +val releaseVersion = "20.5.0" val jacksonVersion = "2.11.0" val jacksonLibs = Seq( From 696076263178ffb99d4e61d314e49bb8710c74e3 Mon Sep 17 00:00:00 2001 From: Ian Bennett Date: Tue, 19 May 2020 01:17:18 +0000 Subject: [PATCH 167/409] Twitter-oss: update OSS libraries post-release to 20.6.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 20.6.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. JIRA Issues: CSL-9788 Differential Revision: https://phabricator.twitter.biz/D486635 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7ff7d5c9..5baf251e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=master)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 90ac53e1..ee3cd77f 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.5.0" +val releaseVersion = "20.6.0-SNAPSHOT" val jacksonVersion = "2.11.0" val jacksonLibs = Seq( From 71a5122542f6f4b920b7b552e2ff8dcc158a7da0 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Wed, 27 May 2020 22:25:09 +0000 Subject: [PATCH 168/409] twitter-server: Add BUILD file for slf4j jdk14 Handler Problem / Solution The directories underneath Twitter Server's `slf4j-jdk14` directory do not follow Pants' recommendation that a single folder and single package have a single BUILD file. This change adds a BUILD file for the `com.twitter.server.handler.slf4j.jdk14` namespace. Differential Revision: https://phabricator.twitter.biz/D491314 --- .../twitter/server/handler/slf4j/jdk14/BUILD | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 slf4j-jdk14/src/main/scala/com/twitter/server/handler/slf4j/jdk14/BUILD diff --git a/slf4j-jdk14/src/main/scala/com/twitter/server/handler/slf4j/jdk14/BUILD b/slf4j-jdk14/src/main/scala/com/twitter/server/handler/slf4j/jdk14/BUILD new file mode 100644 index 00000000..69351f97 --- /dev/null +++ b/slf4j-jdk14/src/main/scala/com/twitter/server/handler/slf4j/jdk14/BUILD @@ -0,0 +1,17 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + provides = scala_artifact( + org = "com.twitter", + name = "twitter-server-slf4j-jdk14-handler", + repo = artifactory, + ), + strict_deps = True, + dependencies = [ + "finagle/finagle-http/src/main/scala", + "twitter-server/server/src/main/scala", + "util/util-core/src/main/scala/com/twitter/io", + "util/util-logging/src/main/scala/com/twitter/logging", + ], + exports = [], +) From 6f31e575dc07671f9208d34fbc714b547cb84bf0 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Thu, 28 May 2020 20:36:13 +0000 Subject: [PATCH 169/409] twitter-server: Add BUILD Files for slf4j jdk14 Components Problem / Solution The directories underneath Twitter Server's slf4j-jdk14 directory do not follow Pants' recommendation that a single folder and single package have a single BUILD file. This change adds BUILD files for the `com.twitter.server.logging` and `com.twitter.server.slf4j.jdk14` namespaces. Differential Revision: https://phabricator.twitter.biz/D492049 --- .../twitter/server/handler/slf4j/jdk14/BUILD | 1 - .../main/scala/com/twitter/server/logging/BUILD | 16 ++++++++++++++++ .../scala/com/twitter/server/slf4j/jdk14/BUILD | 17 +++++++++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 slf4j-jdk14/src/main/scala/com/twitter/server/logging/BUILD create mode 100644 slf4j-jdk14/src/main/scala/com/twitter/server/slf4j/jdk14/BUILD diff --git a/slf4j-jdk14/src/main/scala/com/twitter/server/handler/slf4j/jdk14/BUILD b/slf4j-jdk14/src/main/scala/com/twitter/server/handler/slf4j/jdk14/BUILD index 69351f97..a941f6fb 100644 --- a/slf4j-jdk14/src/main/scala/com/twitter/server/handler/slf4j/jdk14/BUILD +++ b/slf4j-jdk14/src/main/scala/com/twitter/server/handler/slf4j/jdk14/BUILD @@ -13,5 +13,4 @@ scala_library( "util/util-core/src/main/scala/com/twitter/io", "util/util-logging/src/main/scala/com/twitter/logging", ], - exports = [], ) diff --git a/slf4j-jdk14/src/main/scala/com/twitter/server/logging/BUILD b/slf4j-jdk14/src/main/scala/com/twitter/server/logging/BUILD new file mode 100644 index 00000000..0a75a117 --- /dev/null +++ b/slf4j-jdk14/src/main/scala/com/twitter/server/logging/BUILD @@ -0,0 +1,16 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + provides = scala_artifact( + org = "com.twitter", + name = "twitter-server-slf4j-jdk14-logging", + repo = artifactory, + ), + strict_deps = True, + dependencies = [ + "finagle/finagle-core", + "util/util-app/src/main/scala", + "util/util-core:util-core-util", + "util/util-logging/src/main/scala/com/twitter/logging", + ], +) diff --git a/slf4j-jdk14/src/main/scala/com/twitter/server/slf4j/jdk14/BUILD b/slf4j-jdk14/src/main/scala/com/twitter/server/slf4j/jdk14/BUILD new file mode 100644 index 00000000..d71dccf4 --- /dev/null +++ b/slf4j-jdk14/src/main/scala/com/twitter/server/slf4j/jdk14/BUILD @@ -0,0 +1,17 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + provides = scala_artifact( + org = "com.twitter", + name = "twitter-server-slf4j-jdk14", + repo = artifactory, + ), + strict_deps = True, + dependencies = [ + "twitter-server/server/src/main/scala", + "twitter-server/slf4j-jdk14/src/main/scala/com/twitter/server/logging", + ], + exports = [ + "twitter-server/slf4j-jdk14/src/main/scala/com/twitter/server/logging", + ], +) From e64bbd769741f49dc14929dcefe8c00eb6cddfa8 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Thu, 28 May 2020 21:42:10 +0000 Subject: [PATCH 170/409] util|scrooge|finagle|twitter-server|finatra: Use `skip in publish` Problem sbt introduced `skip in publish` here: https://github.com/sbt/sbt/issues/3136, which replaces the need to configure multiple different settings in attempt to skip publishing of a sub-project within a multi-module project. The current workarounds to skip publishing of a sub-project appear to lack some efficacy in practice. Solution Switch to using `skip in publish` and also configure local overwrite to true to aid in release publishing. Differential Revision: https://phabricator.twitter.biz/D491635 --- build.sbt | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/build.sbt b/build.sbt index ee3cd77f..d26e000f 100644 --- a/build.sbt +++ b/build.sbt @@ -16,11 +16,7 @@ def util(which: String) = "com.twitter" %% ("util-" + which) % releaseVersion def finagle(which: String) = "com.twitter" %% ("finagle-" + which) % releaseVersion lazy val noPublishSettings = Seq( - publish := {}, - publishLocal := {}, - publishArtifact := false, - // sbt-pgp's publishSigned task needs this defined even though it is not publishing. - publishTo := Some(Resolver.file("Unused transient repository", file("target/unusedrepo"))) + skip in publish := true ) def gcJavaOptions: Seq[String] = { @@ -124,6 +120,7 @@ lazy val sharedSettings = Seq( pomIncludeRepository := { _ => false }, publishMavenStyle := true, publishConfiguration := publishConfiguration.value.withOverwrite(true), + publishLocalConfiguration := publishLocalConfiguration.value.withOverwrite(true), autoAPIMappings := true, apiURL := Some(url("https://twitter.github.io/twitter-server/docs/")), pomExtra := From 3f36e0c87bf920fd87909642d62ce847b9a1adc0 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Wed, 10 Jun 2020 19:01:56 +0000 Subject: [PATCH 171/409] finagle, finatra, ts: Keep `includecode` Up To Date Across Projects Problem / Solution Recent changes to the Scrooge documentation led to modifying that project's `includecode` custom Sphinx directive. This commit keeps the copies in finagle, finatra, and twitter-server in sync with the latest version from scrooge. Differential Revision: https://phabricator.twitter.biz/D498681 --- doc/src/sphinx/exts/includecode.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/doc/src/sphinx/exts/includecode.py b/doc/src/sphinx/exts/includecode.py index 47d08b6a..3d02b26c 100644 --- a/doc/src/sphinx/exts/includecode.py +++ b/doc/src/sphinx/exts/includecode.py @@ -17,6 +17,7 @@ class IncludeCode(Directive): option_spec = { 'section': directives.unchanged_required, 'comment': directives.unchanged_required, + 'endcomment': directives.unchanged_required, 'marker': directives.unchanged_required, 'include': directives.unchanged_required, 'exclude': directives.unchanged_required, @@ -68,8 +69,11 @@ def run(self): (encoding, filename))] comment = self.options.get('comment', '//') + endcomment = self.options.get('endcomment', '') marker = self.options.get('marker', comment + '#') - lenm = len(marker) + len_marker = len(marker) + len_end_comment = len(endcomment) + if not section: section = self.options.get('section') include_sections = self.options.get('include', '') @@ -85,7 +89,12 @@ def run(self): for line in lines: index = line.find(marker) if index >= 0: - section_name = line[index+lenm:].strip() + start_section = index + len_marker + if len_end_comment > 0: + end_section = -1 * (len_end_comment + 1) # + 1 for newline + section_name = line[start_section:end_section].strip() + else: + section_name = line[start_section:].strip() if section_name in within: within ^= set([section_name]) if excluding and not (exclude & within): From 23f2d5026552216b8d01c4683cce701d0984ab4c Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Sat, 13 Jun 2020 06:03:30 +0000 Subject: [PATCH 172/409] twitter-server: Allow AdminHttpServer to be started later Problem Finatra needs the `AdminHttpServer` to delay startup as it needs to properly provide the `StatsReceiver` from the object graph to the admin server for providing admin links for Histograms. Solution Allow for a [twitter] subclass to set `eagerlyStartAdminHttpServer` to false which will *not* start the AdminHttpServer, forcing the subclass to call `startAdminHttpServer` when appropriate. Note, we also update to not add admin routes to the HttpMuxer when the `disableAdminHttpServer` is true. Differential Revision: https://phabricator.twitter.biz/D500652 --- CHANGELOG.rst | 3 + .../com/twitter/server/AdminHttpServer.scala | 78 +++++++++------- .../twitter/server/AdminHttpServerTest.scala | 93 +++++++++++++++++++ 3 files changed, 138 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4379ed2e..b4dc3002 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +* Allow the `AdminHttpServer` to be started later in the server lifecycle. ``PHAB_ID=D500652`` + + 20.5.0 ------ diff --git a/server/src/main/scala/com/twitter/server/AdminHttpServer.scala b/server/src/main/scala/com/twitter/server/AdminHttpServer.scala index 82c1e630..d64caf48 100644 --- a/server/src/main/scala/com/twitter/server/AdminHttpServer.scala +++ b/server/src/main/scala/com/twitter/server/AdminHttpServer.scala @@ -129,6 +129,14 @@ trait AdminHttpServer { self: App with Stats => */ protected def disableAdminHttpServer: Boolean = false + /** + * By default the Twitter-Server admin will be started in premain -- however there are cases where + * the admin server should be started later. Setting this to false will cause the + * admin server to never be started -- thus the [[startAdminHttpServer()]] should be + * called manually at the desired time to start the admin server. + */ + protected[twitter] def eagerlyStartAdminHttpServer: Boolean = true + def defaultAdminPort: Int = 9990 val adminPort: Flag[InetSocketAddress] = flag("admin.port", new InetSocketAddress(defaultAdminPort), "Admin http server port") @@ -220,6 +228,39 @@ trait AdminHttpServer { self: App with Stats => */ protected def configureAdminHttpServer(server: Http.Server): Http.Server = server + /** Start the AdminHttpServer if [[disableAdminHttpServer]] equals false. */ + protected[twitter] def startAdminHttpServer(): Unit = { + // we delay this check until we call the premain to ensure the `disableAdminHttpServer` value + // has the correct initialization order + if (disableAdminHttpServer) { + log.info("admin http is disabled and will not be started.") + } else { + // add the admin routes + addAdminRoutes(Admin.adminRoutes(statsReceiver, self)) + + log.info(s"Serving admin http on ${adminPort()}") + adminHttpServer = configureAdminHttpServer( + Http.server + .configured(Http.Netty4Impl) + .withStatsReceiver(NullStatsReceiver) + .withTracer(NullTracer) + .withMonitor(new Monitor { + def handle(exc: Throwable): Boolean = { + log.error(s"Caught exception in AdminHttpServer: $exc", exc) + false + } + }) + .withLabel(ServerName) + // disable admission control, since we want the server to report stats + // especially when it's in a bad state. + .configured(ServerAdmissionControl.Param(false)) + ).serve(adminPort(), new NotFoundView andThen adminHttpMuxer) + + closeOnExitLast(adminHttpServer) + Library.register(libraryName, Map.empty) + } + } + private[this] def updateMuxer(): Unit = { // create a service which multiplexes across all endpoints. val localMuxer = allRoutes.foldLeft(new HttpMuxer) { @@ -281,42 +322,7 @@ trait AdminHttpServer { self: App with Stats => private[this] def routeToIndexLink(route: Route): IndexView.Link = IndexView.Link(route.alias, route.path, route.method) - private[this] def startServer(): Unit = { - val loggingMonitor = new Monitor { - def handle(exc: Throwable): Boolean = { - log.error(s"Caught exception in AdminHttpServer: $exc", exc) - false - } - } - - log.info(s"Serving admin http on ${adminPort()}") - adminHttpServer = configureAdminHttpServer( - Http.server - .configured(Http.Netty4Impl) - .withStatsReceiver(NullStatsReceiver) - .withTracer(NullTracer) - .withMonitor(loggingMonitor) - .withLabel(ServerName) - // disable admission control, since we want the server to report stats - // especially when it's in a bad state. - .configured(ServerAdmissionControl.Param(false)) - ).serve(adminPort(), new NotFoundView andThen adminHttpMuxer) - - closeOnExitLast(adminHttpServer) - Library.register(libraryName, Map.empty) - } - premain { - // For consistency, we will add the routes regardless of whether the `adminHttpServer` gets - // started. This may not always be true and we may change this behavior in the future. - addAdminRoutes(Admin.adminRoutes(statsReceiver, self)) - - // we delay this check until we call the premain to ensure the `disableAdminHttpServer` value - // has the correct initialization order - if (disableAdminHttpServer) { - log.info("admin http is disabled and will not be started.") - } else { - startServer() - } + if (eagerlyStartAdminHttpServer) startAdminHttpServer() } } diff --git a/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala b/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala index e8871553..1119afad 100644 --- a/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala +++ b/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala @@ -3,6 +3,7 @@ package com.twitter.server import com.twitter.conversions.DurationOps._ import com.twitter.finagle.{Http, ListeningServer} import com.twitter.finagle.http._ +import com.twitter.finagle.stats.NullStatsReceiver import com.twitter.server.util.HttpUtils._ import com.twitter.util._ import java.net.InetSocketAddress @@ -147,6 +148,98 @@ class AdminHttpServerTest extends FunSuite with Eventually with IntegrationPatie } server.main(args = Array.empty[String]) assert(MockClosableHandler.closed) + // assert routes are not added when the admin is disabled + val adminRoutes = Admin.adminRoutes(NullStatsReceiver, server) + assert(server.routes.map(_.path).intersect(adminRoutes.map(_.path)).isEmpty) + } + + test("a non eagerly started server does not start") { + val server = new TestTwitterServer { + override protected[twitter] def eagerlyStartAdminHttpServer: Boolean = false + override def main(): Unit = { + intercept[Exception] { + checkServer(adminHttpServer) + } + + val adminServerBoundPort = + adminHttpServer.boundAddress.asInstanceOf[InetSocketAddress].getPort + assert(adminServerBoundPort == adminBoundAddress.getPort) + val client = Http.client.newService(s"localhost:$adminServerBoundPort") + intercept[Exception] { // the admin is already closed, this should error + Await.result(client(Request(Method.Post, "/quitquitquit")), 1.second) + } + + Await.result(close(5.seconds), 7.seconds) + } + } + server.main(args = Array.empty[String]) + assert(MockClosableHandler.closed) + // assert routes are not added as the admin is not started + val adminRoutes = Admin.adminRoutes(NullStatsReceiver, server) + assert(server.routes.map(_.path).intersect(adminRoutes.map(_.path)).isEmpty) + } + + test("a non eagerly started server can be started explicitly") { + val server = new TestTwitterServer { + override protected[twitter] def eagerlyStartAdminHttpServer: Boolean = false + + premain { + // admin server has not yet been started + intercept[Exception] { + checkServer(adminHttpServer) + } + } + + override def main(): Unit = { + startAdminHttpServer() + checkServer(adminHttpServer) + + closeServer(this, adminHttpServer) + Await.result(close(5.seconds), 7.seconds) + } + } + server.main(args = Array.empty[String]) + assert(MockClosableHandler.closed) + // assert routes are added + val adminRoutes = Admin.adminRoutes(NullStatsReceiver, server) + assert(server.routes.map(_.path).intersect(adminRoutes.map(_.path)).nonEmpty) + } + + test("a non eagerly started server still doesn't start when disabled") { + val server = new TestTwitterServer { + override protected def disableAdminHttpServer: Boolean = true + override protected[twitter] def eagerlyStartAdminHttpServer: Boolean = false + + premain { + // admin server has not yet been started + intercept[Exception] { + checkServer(adminHttpServer) + } + } + + override def main(): Unit = { + startAdminHttpServer() + // still disabled + intercept[Exception] { + checkServer(adminHttpServer) + } + + val adminServerBoundPort = + adminHttpServer.boundAddress.asInstanceOf[InetSocketAddress].getPort + assert(adminServerBoundPort == adminBoundAddress.getPort) + val client = Http.client.newService(s"localhost:$adminServerBoundPort") + intercept[Exception] { // the admin is already closed, this should error + Await.result(client(Request(Method.Post, "/quitquitquit")), 1.second) + } + + Await.result(close(5.seconds), 7.seconds) + } + } + server.main(args = Array.empty[String]) + assert(MockClosableHandler.closed) + // assert routes are not added when the admin is disabled + val adminRoutes = Admin.adminRoutes(NullStatsReceiver, server) + assert(server.routes.map(_.path).intersect(adminRoutes.map(_.path)).isEmpty) } test("GET does not close server") { From 81a9fe9890fa3a31b4487c353de7b43d9f51e4ab Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Mon, 15 Jun 2020 17:42:55 +0000 Subject: [PATCH 173/409] Revert 'Allow AdminHttpServer to be started later' This reverts commit d99e96d0a7a284d11dc3fdb99030c42c24dbe83c. These changes were to help solve a lifecycle impedance mismatch between `c.t.server.Stats#statsReceiver` and the Finatra framework. However, it seems that solving that mismatch will be nigh impossible for legacy reasons and because the hacks to do so make usage of the framework very brittle and prone to error -- e.g., the path of the current solution would mean users would need to know that accessing the `c.t.server.Stats#statsReceiver` should only be done at a certain point in the lifecycle. We will likely need to resort to documenting the issue and providing clarity on how to deal with the inconsistencies over a real fix. Since these AdminHttpServer changes were meant to help solve the lifecycle issue, it is better to revert them since they will not be used and may cause more harm than good if left. Differential Revision: https://phabricator.twitter.biz/D500871 --- CHANGELOG.rst | 3 - .../com/twitter/server/AdminHttpServer.scala | 78 +++++++--------- .../twitter/server/AdminHttpServerTest.scala | 93 ------------------- 3 files changed, 36 insertions(+), 138 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b4dc3002..4379ed2e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,9 +7,6 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- -* Allow the `AdminHttpServer` to be started later in the server lifecycle. ``PHAB_ID=D500652`` - - 20.5.0 ------ diff --git a/server/src/main/scala/com/twitter/server/AdminHttpServer.scala b/server/src/main/scala/com/twitter/server/AdminHttpServer.scala index d64caf48..82c1e630 100644 --- a/server/src/main/scala/com/twitter/server/AdminHttpServer.scala +++ b/server/src/main/scala/com/twitter/server/AdminHttpServer.scala @@ -129,14 +129,6 @@ trait AdminHttpServer { self: App with Stats => */ protected def disableAdminHttpServer: Boolean = false - /** - * By default the Twitter-Server admin will be started in premain -- however there are cases where - * the admin server should be started later. Setting this to false will cause the - * admin server to never be started -- thus the [[startAdminHttpServer()]] should be - * called manually at the desired time to start the admin server. - */ - protected[twitter] def eagerlyStartAdminHttpServer: Boolean = true - def defaultAdminPort: Int = 9990 val adminPort: Flag[InetSocketAddress] = flag("admin.port", new InetSocketAddress(defaultAdminPort), "Admin http server port") @@ -228,39 +220,6 @@ trait AdminHttpServer { self: App with Stats => */ protected def configureAdminHttpServer(server: Http.Server): Http.Server = server - /** Start the AdminHttpServer if [[disableAdminHttpServer]] equals false. */ - protected[twitter] def startAdminHttpServer(): Unit = { - // we delay this check until we call the premain to ensure the `disableAdminHttpServer` value - // has the correct initialization order - if (disableAdminHttpServer) { - log.info("admin http is disabled and will not be started.") - } else { - // add the admin routes - addAdminRoutes(Admin.adminRoutes(statsReceiver, self)) - - log.info(s"Serving admin http on ${adminPort()}") - adminHttpServer = configureAdminHttpServer( - Http.server - .configured(Http.Netty4Impl) - .withStatsReceiver(NullStatsReceiver) - .withTracer(NullTracer) - .withMonitor(new Monitor { - def handle(exc: Throwable): Boolean = { - log.error(s"Caught exception in AdminHttpServer: $exc", exc) - false - } - }) - .withLabel(ServerName) - // disable admission control, since we want the server to report stats - // especially when it's in a bad state. - .configured(ServerAdmissionControl.Param(false)) - ).serve(adminPort(), new NotFoundView andThen adminHttpMuxer) - - closeOnExitLast(adminHttpServer) - Library.register(libraryName, Map.empty) - } - } - private[this] def updateMuxer(): Unit = { // create a service which multiplexes across all endpoints. val localMuxer = allRoutes.foldLeft(new HttpMuxer) { @@ -322,7 +281,42 @@ trait AdminHttpServer { self: App with Stats => private[this] def routeToIndexLink(route: Route): IndexView.Link = IndexView.Link(route.alias, route.path, route.method) + private[this] def startServer(): Unit = { + val loggingMonitor = new Monitor { + def handle(exc: Throwable): Boolean = { + log.error(s"Caught exception in AdminHttpServer: $exc", exc) + false + } + } + + log.info(s"Serving admin http on ${adminPort()}") + adminHttpServer = configureAdminHttpServer( + Http.server + .configured(Http.Netty4Impl) + .withStatsReceiver(NullStatsReceiver) + .withTracer(NullTracer) + .withMonitor(loggingMonitor) + .withLabel(ServerName) + // disable admission control, since we want the server to report stats + // especially when it's in a bad state. + .configured(ServerAdmissionControl.Param(false)) + ).serve(adminPort(), new NotFoundView andThen adminHttpMuxer) + + closeOnExitLast(adminHttpServer) + Library.register(libraryName, Map.empty) + } + premain { - if (eagerlyStartAdminHttpServer) startAdminHttpServer() + // For consistency, we will add the routes regardless of whether the `adminHttpServer` gets + // started. This may not always be true and we may change this behavior in the future. + addAdminRoutes(Admin.adminRoutes(statsReceiver, self)) + + // we delay this check until we call the premain to ensure the `disableAdminHttpServer` value + // has the correct initialization order + if (disableAdminHttpServer) { + log.info("admin http is disabled and will not be started.") + } else { + startServer() + } } } diff --git a/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala b/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala index 1119afad..e8871553 100644 --- a/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala +++ b/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala @@ -3,7 +3,6 @@ package com.twitter.server import com.twitter.conversions.DurationOps._ import com.twitter.finagle.{Http, ListeningServer} import com.twitter.finagle.http._ -import com.twitter.finagle.stats.NullStatsReceiver import com.twitter.server.util.HttpUtils._ import com.twitter.util._ import java.net.InetSocketAddress @@ -148,98 +147,6 @@ class AdminHttpServerTest extends FunSuite with Eventually with IntegrationPatie } server.main(args = Array.empty[String]) assert(MockClosableHandler.closed) - // assert routes are not added when the admin is disabled - val adminRoutes = Admin.adminRoutes(NullStatsReceiver, server) - assert(server.routes.map(_.path).intersect(adminRoutes.map(_.path)).isEmpty) - } - - test("a non eagerly started server does not start") { - val server = new TestTwitterServer { - override protected[twitter] def eagerlyStartAdminHttpServer: Boolean = false - override def main(): Unit = { - intercept[Exception] { - checkServer(adminHttpServer) - } - - val adminServerBoundPort = - adminHttpServer.boundAddress.asInstanceOf[InetSocketAddress].getPort - assert(adminServerBoundPort == adminBoundAddress.getPort) - val client = Http.client.newService(s"localhost:$adminServerBoundPort") - intercept[Exception] { // the admin is already closed, this should error - Await.result(client(Request(Method.Post, "/quitquitquit")), 1.second) - } - - Await.result(close(5.seconds), 7.seconds) - } - } - server.main(args = Array.empty[String]) - assert(MockClosableHandler.closed) - // assert routes are not added as the admin is not started - val adminRoutes = Admin.adminRoutes(NullStatsReceiver, server) - assert(server.routes.map(_.path).intersect(adminRoutes.map(_.path)).isEmpty) - } - - test("a non eagerly started server can be started explicitly") { - val server = new TestTwitterServer { - override protected[twitter] def eagerlyStartAdminHttpServer: Boolean = false - - premain { - // admin server has not yet been started - intercept[Exception] { - checkServer(adminHttpServer) - } - } - - override def main(): Unit = { - startAdminHttpServer() - checkServer(adminHttpServer) - - closeServer(this, adminHttpServer) - Await.result(close(5.seconds), 7.seconds) - } - } - server.main(args = Array.empty[String]) - assert(MockClosableHandler.closed) - // assert routes are added - val adminRoutes = Admin.adminRoutes(NullStatsReceiver, server) - assert(server.routes.map(_.path).intersect(adminRoutes.map(_.path)).nonEmpty) - } - - test("a non eagerly started server still doesn't start when disabled") { - val server = new TestTwitterServer { - override protected def disableAdminHttpServer: Boolean = true - override protected[twitter] def eagerlyStartAdminHttpServer: Boolean = false - - premain { - // admin server has not yet been started - intercept[Exception] { - checkServer(adminHttpServer) - } - } - - override def main(): Unit = { - startAdminHttpServer() - // still disabled - intercept[Exception] { - checkServer(adminHttpServer) - } - - val adminServerBoundPort = - adminHttpServer.boundAddress.asInstanceOf[InetSocketAddress].getPort - assert(adminServerBoundPort == adminBoundAddress.getPort) - val client = Http.client.newService(s"localhost:$adminServerBoundPort") - intercept[Exception] { // the admin is already closed, this should error - Await.result(client(Request(Method.Post, "/quitquitquit")), 1.second) - } - - Await.result(close(5.seconds), 7.seconds) - } - } - server.main(args = Array.empty[String]) - assert(MockClosableHandler.closed) - // assert routes are not added when the admin is disabled - val adminRoutes = Admin.adminRoutes(NullStatsReceiver, server) - assert(server.routes.map(_.path).intersect(adminRoutes.map(_.path)).isEmpty) } test("GET does not close server") { From b1f97febfe09f9e4234c99ff04b66a6980670528 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Tue, 16 Jun 2020 01:21:52 +0000 Subject: [PATCH 174/409] util|scrooge|finagle|twitter-server|finatra: Update TravisCI badge links Problem/Solution We recently moved the projects from `travis-ci.org` to `travis-ci.com` and need to now update the badge links. Differential Revision: https://phabricator.twitter.biz/D501490 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5baf251e..d783ddb3 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.org/twitter/twitter-server.svg?branch=develop)](https://travis-ci.org/twitter/twitter-server) +[![Build status](https://travis-ci.com/twitter/twitter-server.svg?branch=develop)](https://travis-ci.com/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) From 8c01646cb17312dbc9a74d636b76a181a3234838 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Fri, 19 Jun 2020 03:33:47 +0000 Subject: [PATCH 175/409] twitter-server: Make c.t.server.BuildProperties private[twitter] Problem/Solution Widen visibility for `c.t.server.BuildProperties` to `private[twitter]`. Differential Revision: https://phabricator.twitter.biz/D504033 --- .../com/twitter/server/BuildProperties.scala | 49 ++++++++----------- 1 file changed, 21 insertions(+), 28 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/BuildProperties.scala b/server/src/main/scala/com/twitter/server/BuildProperties.scala index 1241f6ca..b08f1667 100644 --- a/server/src/main/scala/com/twitter/server/BuildProperties.scala +++ b/server/src/main/scala/com/twitter/server/BuildProperties.scala @@ -12,7 +12,7 @@ import scala.util.control.NonFatal * the given object's class's package (class-package-name/build.properties), and if not found there, * then it is searched for with an absolute path ("/build.properties"). */ -private[server] object BuildProperties { +private[twitter] object BuildProperties { private[this] val log = Logger(BuildProperties.getClass) private[this] val basicServerInfo: Map[String, String] = @@ -27,26 +27,25 @@ private[server] object BuildProperties { "scm_repository" -> "unknown" ) - private[this] val buildProperties = new Properties - try { - buildProperties.load(BuildProperties.getClass.getResource("build.properties").openStream) - } catch { - case NonFatal(_) => - try { - BuildProperties.getClass.getResource("/build.properties") match { - case resource: URL => - buildProperties.load(resource.openStream) - case _ => // do nothing + private[this] val properties: Map[String, String] = { + val buildProperties = new Properties + try { + buildProperties.load(BuildProperties.getClass.getResource("build.properties").openStream) + } catch { + case NonFatal(_) => + try { + BuildProperties.getClass.getResource("/build.properties") match { + case resource: URL => + buildProperties.load(resource.openStream) + case _ => // do nothing + } + } catch { + case NonFatal(e) => + log.warn("Unable to load build.properties file from classpath. " + e.getMessage) } - } catch { - case NonFatal(e) => - log.warn("Unable to load build.properties file from classpath. " + e.getMessage) - } - } - - /* Lazy to allow for loading of the buildProperties before this val is read */ - private[this] lazy val combinedInfo: Map[String, String] = + } basicServerInfo ++ buildProperties.asScala + } /** * Returns the [[String]] value associated with this key or a `NoSuchElementException` if there @@ -55,9 +54,7 @@ private[server] object BuildProperties { * @param key the key * @return the value associated with the given key, or a `NoSuchElementException`. */ - def get(key: String): String = { - all(key) - } + def get(key: String): String = all(key) /** * Returns the value associated with a key, or a default value if the key is not contained in the map. @@ -66,14 +63,10 @@ private[server] object BuildProperties { * @param defaultValue a default value in case no binding for `key` is found in the map. * @return the value associated with `key` if it exists, otherwise the `defaultValue`. */ - def get(key: String, defaultValue: String): String = { - all.getOrElse(key, defaultValue) - } + def get(key: String, defaultValue: String): String = all.getOrElse(key, defaultValue) /** * Return all build properties. */ - def all: Map[String, String] = { - combinedInfo - } + def all: Map[String, String] = properties } From 971c9a7ea5a0968ee6fa81aa85a4a15677969e90 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Tue, 23 Jun 2020 16:43:45 +0000 Subject: [PATCH 176/409] util|scrooge|finagle|twitter-server|finatra - Update READMEs Problem/Solution Ensure all the README files have a license with copyright date section. Update the date to 2020 (finally). Fix up Finatra Thrift example to the latest code. Differential Revision: https://phabricator.twitter.biz/D505486 --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index d783ddb3..0735b712 100644 --- a/README.md +++ b/README.md @@ -45,3 +45,9 @@ TwitterServer, and weekly snapshots are published to the `develop` branch. In ge pull requests should be submitted against `develop`. See [CONTRIBUTING.md](https://github.com/twitter/twitter-server/blob/master/CONTRIBUTING.md) for more details about how to contribute. + +## License + +Copyright 2013-2020 Twitter, Inc. + +Licensed under the Apache License, Version 2.0: https://www.apache.org/licenses/LICENSE-2.0 From e8d39b88f8d6506e2712f1fd5e9b16aa544d1dbc Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Tue, 23 Jun 2020 23:58:54 +0000 Subject: [PATCH 177/409] twitter-oss: Prepare OSS libraries for release 20.6.0 Problem We want to release the next versions of our Twitter OSS libraries 20.6.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D506048 --- CHANGELOG.rst | 5 +++++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4379ed2e..b1fec7fb 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +20.6.0 +------ + +No Changes + 20.5.0 ------ diff --git a/README.md b/README.md index 0735b712..88e9fbeb 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.com/twitter/twitter-server.svg?branch=develop)](https://travis-ci.com/twitter/twitter-server) +[![Build status](https://travis-ci.com/twitter/twitter-server.svg?branch=master)](https://travis-ci.com/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index d26e000f..b9b52feb 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.6.0-SNAPSHOT" +val releaseVersion = "20.6.0" val jacksonVersion = "2.11.0" val jacksonLibs = Seq( From d7cd03237ea8976c779e23f92fe65557515603b0 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Wed, 24 Jun 2020 01:05:30 +0000 Subject: [PATCH 178/409] twitter-oss: Update OSS Libraries Post-Release to 20.7.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 20.7.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D506108 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 88e9fbeb..0735b712 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.com/twitter/twitter-server.svg?branch=master)](https://travis-ci.com/twitter/twitter-server) +[![Build status](https://travis-ci.com/twitter/twitter-server.svg?branch=develop)](https://travis-ci.com/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index b9b52feb..9615144e 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.6.0" +val releaseVersion = "20.7.0-SNAPSHOT" val jacksonVersion = "2.11.0" val jacksonLibs = Seq( From cb85a45fbc4d4d380b66f14db16b164a4d194981 Mon Sep 17 00:00:00 2001 From: Ian Bennett Date: Fri, 26 Jun 2020 00:09:41 +0000 Subject: [PATCH 179/409] twitter-server: make Lifecycle and Warmup self-typed to TwitterServer Problem/Solution The `Lifecycle` trait is self-typed to `c.t.app.App`, when it is only used in the context of `TwitterServer`. Additionally, the `Lifecycle.Warmup` trait has no type restrictions, despite the intention of only working within the context of a `TwitterServer`. Let's change the self-type restriction for both to `TwitterServer`. We create a `Lifecycle.DetatchedWarmup` trait to temporarily allow continued `Lifecycle.Warmup` behavior for users who were previously using `Lifecycle.Warmup` trait without mixing into a `TwitterServer`. This behavior will be removed in a future release. JIRA Issues: CSL-9907 Differential Revision: https://phabricator.twitter.biz/D507392 --- CHANGELOG.rst | 10 +++ .../scala/com/twitter/server/Lifecycle.scala | 61 ++++++++++++++++--- 2 files changed, 61 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b1fec7fb..a55f4f5e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,16 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Breaking API Changes +~~~~~~~~~~~~~~~~~~~~ + +* Make `Lifecycle` and `Lifecycle.Warmup` self-typed to `TwitterServer`. `Lifecycle` was previously + self-typed to `c.t.app.App` and `Lifecycle.Warmup` previously had no self-type restrictions. These + traits can now only be mixed into instances of `TwitterServer`. The `Lifecycle.DetatchedWarmup` + trait is introduced to allow users to transition to it, where they were previously extending + `Lifecycle.Warmup` without mixing into a `TwitterServer`. `Lifecycle.DetatchedWarmup` + is immediately deprecated and will be removed in a future release. ``PHAB_ID=D507392`` + 20.6.0 ------ diff --git a/server/src/main/scala/com/twitter/server/Lifecycle.scala b/server/src/main/scala/com/twitter/server/Lifecycle.scala index fc8268ec..0f1fa8e5 100644 --- a/server/src/main/scala/com/twitter/server/Lifecycle.scala +++ b/server/src/main/scala/com/twitter/server/Lifecycle.scala @@ -1,6 +1,6 @@ package com.twitter.server -import com.twitter.app.{App, GlobalFlag} +import com.twitter.app.GlobalFlag import com.twitter.finagle.http.Method.Post import com.twitter.finagle.http.{HttpMuxer, Route, RouteIndex} import com.twitter.server.handler._ @@ -8,7 +8,7 @@ import java.lang.management.ManagementFactory import java.util.concurrent.atomic.AtomicBoolean import scala.collection.JavaConverters._ -trait Lifecycle { self: App => +trait Lifecycle { self: TwitterServer => // Mesos/Aurora lifecycle endpoints val group = "Misc" HttpMuxer.addHandler( @@ -76,6 +76,51 @@ object Lifecycle { } } + private[Lifecycle] object Warmup { + + /** + * Initialize warmup code. Ensures that the /health endpoint will not return on "OK" response. + */ + def initializeWarmup(): Unit = + HttpMuxer.addHandler(Route("/health", new ReplyHandler(""))) + + /** + * Prebind warmup code. Used for warmup tasks that we want to run before we + * accept traffic. + */ + def prebindWarmup(): Unit = new PromoteToOldGen().beforeServing() + + /** + * The service is bound to a port and warmed up, announce health. + */ + def warmupComplete(): Unit = + HttpMuxer.addHandler(Route("/health", new ReplyHandler("OK\n"))) + + } + + /** + * A [[Warmup]] that is detached from a [[TwitterServer]] lifecycle. As there is no guarantee that + * this trait is used within the context of a [[TwitterServer]], the behavior and expectations of + * this warmup are determined by the implementor. + */ + @deprecated( + "Warmup behavior is a TwitterServer lifecycle concern. Please mixin Warmup to your TwitterServer.", + "2020-06-25") + trait DetatchedWarmup { + Warmup.initializeWarmup() + + /** + * Prebind warmup code. Used for warmup tasks that we want to run before we + * accept traffic. + */ + def prebindWarmup(): Unit = Warmup.prebindWarmup() + + /** + * The service is bound to a port and warmed up, announce health. + */ + def warmupComplete(): Unit = Warmup.warmupComplete() + } + /** * Give the application control over when to present to Mesos as being ready * for traffic. When the method `warmupComplete()` is invoked, the application @@ -83,22 +128,18 @@ object Lifecycle { * @note Mesos doesn't gate traffic on /health so all pre-bind warmup needs to * happen in `prebindWarmup()` */ - trait Warmup { - HttpMuxer.addHandler(Route("/health", new ReplyHandler(""))) + trait Warmup { self: TwitterServer => + Warmup.initializeWarmup() /** * Prebind warmup code. Used for warmup tasks that we want to run before we * accept traffic. */ - def prebindWarmup(): Unit = { - new PromoteToOldGen().beforeServing() - } + def prebindWarmup(): Unit = Warmup.prebindWarmup() /** * The service is bound to a port and warmed up, announce health. */ - def warmupComplete(): Unit = { - HttpMuxer.addHandler(Route("/health", new ReplyHandler("OK\n"))) - } + def warmupComplete(): Unit = Warmup.warmupComplete() } } From 1efdef9316a3fa12961afa57d90f0edc6b03e3b3 Mon Sep 17 00:00:00 2001 From: Ian Bennett Date: Wed, 1 Jul 2020 17:09:50 +0000 Subject: [PATCH 180/409] twitter-server: expose startupCompletionEvent for TwitterServer's lifecycle Problem/Solution `c.t.app.App` now supports emitting lifecycle events to observers. `c.t.server.Server` should also emit events for `Lifecycle.Warmup` and expose a `startupCompletionEvent` for determining when a server has been marked as started, given varying lifecycle attributes. These changes are all scoped to `private[twitter]` and primarily driven by speeding up our test utilities in the Twitter-stack. JIRA Issues: CSL-9910 Differential Revision: https://phabricator.twitter.biz/D510189 --- .../main/scala/com/twitter/server/Lifecycle.scala | 12 ++++++++++-- .../scala/com/twitter/server/TwitterServer.scala | 6 ++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/Lifecycle.scala b/server/src/main/scala/com/twitter/server/Lifecycle.scala index 0f1fa8e5..9da5a62f 100644 --- a/server/src/main/scala/com/twitter/server/Lifecycle.scala +++ b/server/src/main/scala/com/twitter/server/Lifecycle.scala @@ -1,6 +1,8 @@ package com.twitter.server import com.twitter.app.GlobalFlag +import com.twitter.app.lifecycle.Event +import com.twitter.app.lifecycle.Event.{PrebindWarmup, WarmupComplete} import com.twitter.finagle.http.Method.Post import com.twitter.finagle.http.{HttpMuxer, Route, RouteIndex} import com.twitter.server.handler._ @@ -131,15 +133,21 @@ object Lifecycle { trait Warmup { self: TwitterServer => Warmup.initializeWarmup() + override protected[twitter] def startupCompletionEvent: Event = WarmupComplete + /** * Prebind warmup code. Used for warmup tasks that we want to run before we * accept traffic. */ - def prebindWarmup(): Unit = Warmup.prebindWarmup() + def prebindWarmup(): Unit = observe(PrebindWarmup) { + Warmup.prebindWarmup() + } /** * The service is bound to a port and warmed up, announce health. */ - def warmupComplete(): Unit = Warmup.warmupComplete() + def warmupComplete(): Unit = observe(WarmupComplete) { + Warmup.warmupComplete() + } } } diff --git a/server/src/main/scala/com/twitter/server/TwitterServer.scala b/server/src/main/scala/com/twitter/server/TwitterServer.scala index 514b6f25..f9724ffa 100644 --- a/server/src/main/scala/com/twitter/server/TwitterServer.scala +++ b/server/src/main/scala/com/twitter/server/TwitterServer.scala @@ -1,6 +1,8 @@ package com.twitter.server import com.twitter.app.App +import com.twitter.app.lifecycle.Event +import com.twitter.app.lifecycle.Event.PreMain import com.twitter.finagle.DtabFlags import com.twitter.finagle.util.DefaultTimer import com.twitter.util.Timer @@ -48,6 +50,10 @@ trait TwitterServer /** Use the Finagle DefaultTimer */ override protected lazy val shutdownTimer: Timer = DefaultTimer + + // The `main()` will block until `close()` is initiated, so we need + // to use the last phase that completes prior to Main, which is PreMain. + protected[twitter] def startupCompletionEvent: Event = PreMain } /** From 580390e8daa0ca1a852823b9d6ad6a1f6e4f75c2 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Thu, 16 Jul 2020 21:42:07 +0000 Subject: [PATCH 181/409] dodo et al.: Update Projects to use Scala 2.12.11 Problem The Scala 2.12 version used for CSL's open source projects has diverged from that used internally at Twitter via pants. Solution Update all CSL open source projects to use Scala 2.12.11 to match the recently updated internal version. JIRA Issues: CSL-9990 Differential Revision: https://phabricator.twitter.biz/D518430 --- build.sbt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index 9615144e..07efc89f 100644 --- a/build.sbt +++ b/build.sbt @@ -76,8 +76,8 @@ def travisTestJavaOptions: Seq[String] = { lazy val sharedSettings = Seq( version := releaseVersion, organization := "com.twitter", - scalaVersion := "2.12.8", - crossScalaVersions := Seq("2.11.12", "2.12.8", "2.13.1"), + scalaVersion := "2.12.11", + crossScalaVersions := Seq("2.11.12", "2.12.11", "2.13.1"), fork in Test := true, // We have to fork to get the JavaOptions libraryDependencies ++= Seq( "org.scalacheck" %% "scalacheck" % "1.14.0" % "test", From 24f3ffca6b5d3e52a2cb698628b2962a61d71ddf Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Fri, 17 Jul 2020 01:20:53 +0000 Subject: [PATCH 182/409] 3rdparty: Upgrade scalatest to 3.1.2 Problem / Solution We need to upgrade scalatest, so that we can stay up to date, take advantage of new features, and test our libraries against dotty. This is a more involved upgrade than normal. A lot of old names have been removed, and a bunch of namespaces were moved into new modules, and so we've had to move a lot of things around. TBR=true NO-QUEUE=true NO_USER_HOOK=1 Differential Revision: https://phabricator.twitter.biz/D518553 --- build.sbt | 9 ++++++--- logback-classic/src/test/scala/BUILD | 1 + server/src/test/java/BUILD | 2 ++ server/src/test/scala/BUILD | 3 +++ .../com/twitter/server/handler/TracingHandlerTest.scala | 2 +- .../twitter/server/lint/SchedulerBlockingRuleTest.scala | 2 +- slf4j-jdk14/src/test/scala/BUILD | 2 ++ slf4j-log4j12/src/test/scala/BUILD | 1 + 8 files changed, 17 insertions(+), 5 deletions(-) diff --git a/build.sbt b/build.sbt index 07efc89f..5f6e0307 100644 --- a/build.sbt +++ b/build.sbt @@ -80,11 +80,14 @@ lazy val sharedSettings = Seq( crossScalaVersions := Seq("2.11.12", "2.12.11", "2.13.1"), fork in Test := true, // We have to fork to get the JavaOptions libraryDependencies ++= Seq( - "org.scalacheck" %% "scalacheck" % "1.14.0" % "test", - "org.scalatest" %% "scalatest" % "3.0.8" % "test", // See https://www.scala-sbt.org/0.13/docs/Testing.html#JUnit "com.novocode" % "junit-interface" % "0.11" % "test", - "org.mockito" % "mockito-all" % "1.9.5" % "test" + "org.mockito" % "mockito-all" % "1.9.5" % "test", + "org.scalacheck" %% "scalacheck" % "1.14.3" % "test", + "org.scalatest" %% "scalatest" % "3.1.1" % "test" + "org.scalatestplus" %% "junit" % "3.1.2.0" % "test" + "org.scalatestplus" %% "mockito-1-10" % "3.1.1.0" % "test", + "org.scalatestplus" %% "scalacheck-1-14" % "3.1.2.0" % "test" ), ScoverageKeys.coverageHighlighting := true, ivyXML := diff --git a/logback-classic/src/test/scala/BUILD b/logback-classic/src/test/scala/BUILD index c46ecae9..e54c1ff5 100644 --- a/logback-classic/src/test/scala/BUILD +++ b/logback-classic/src/test/scala/BUILD @@ -5,6 +5,7 @@ junit_tests( "3rdparty/jvm/ch/qos/logback:logback-classic", "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatest", + "3rdparty/jvm/org/scalatestplus:junit", "3rdparty/jvm/org/slf4j:slf4j-api", "finagle/finagle-core/src/main", "twitter-server/logback-classic/src/main/scala", diff --git a/server/src/test/java/BUILD b/server/src/test/java/BUILD index 7da6c4d1..47b5ea2c 100644 --- a/server/src/test/java/BUILD +++ b/server/src/test/java/BUILD @@ -6,6 +6,8 @@ junit_tests( "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scala-lang:scala-library", "3rdparty/jvm/org/scalacheck", + "3rdparty/jvm/org/scalatestplus:junit", + "3rdparty/jvm/org/scalatestplus:mockito-1-10", "3rdparty/jvm/org/slf4j:slf4j-api", "3rdparty/jvm/org/slf4j:slf4j-nop", "finagle/finagle-core/src/main", diff --git a/server/src/test/scala/BUILD b/server/src/test/scala/BUILD index cea41817..71ff0fc9 100644 --- a/server/src/test/scala/BUILD +++ b/server/src/test/scala/BUILD @@ -8,6 +8,9 @@ junit_tests( "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scalacheck", "3rdparty/jvm/org/scalatest", + "3rdparty/jvm/org/scalatestplus:junit", + "3rdparty/jvm/org/scalatestplus:mockito-1-10", + "3rdparty/jvm/org/scalatestplus:scalacheck", "3rdparty/jvm/org/slf4j:slf4j-api", "3rdparty/jvm/org/slf4j:slf4j-nop", "finagle/finagle-core/src/main", diff --git a/server/src/test/scala/com/twitter/server/handler/TracingHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/TracingHandlerTest.scala index bcbb0de0..56c384d1 100644 --- a/server/src/test/scala/com/twitter/server/handler/TracingHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/TracingHandlerTest.scala @@ -4,8 +4,8 @@ import com.twitter.conversions.DurationOps._ import com.twitter.finagle.http.{Request, Status} import com.twitter.finagle.tracing.{Trace, Tracer} import com.twitter.util.{Await, Awaitable} -import org.scalatest.mockito.MockitoSugar import org.scalatest.{BeforeAndAfter, FunSuite} +import org.scalatestplus.mockito.MockitoSugar class TracingHandlerTest extends FunSuite with MockitoSugar with BeforeAndAfter { val service = new TracingHandler diff --git a/server/src/test/scala/com/twitter/server/lint/SchedulerBlockingRuleTest.scala b/server/src/test/scala/com/twitter/server/lint/SchedulerBlockingRuleTest.scala index 3aef6ec4..402d28c9 100644 --- a/server/src/test/scala/com/twitter/server/lint/SchedulerBlockingRuleTest.scala +++ b/server/src/test/scala/com/twitter/server/lint/SchedulerBlockingRuleTest.scala @@ -4,7 +4,7 @@ import com.twitter.concurrent.Scheduler import com.twitter.conversions.DurationOps._ import org.mockito.Mockito._ import org.scalatest.{FunSuite, Matchers} -import org.scalatest.mockito.MockitoSugar +import org.scalatestplus.mockito.MockitoSugar class SchedulerBlockingRuleTest extends FunSuite with Matchers with MockitoSugar { diff --git a/slf4j-jdk14/src/test/scala/BUILD b/slf4j-jdk14/src/test/scala/BUILD index 68132d99..0d4289b0 100644 --- a/slf4j-jdk14/src/test/scala/BUILD +++ b/slf4j-jdk14/src/test/scala/BUILD @@ -6,6 +6,8 @@ junit_tests( "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scalacheck", "3rdparty/jvm/org/scalatest", + "3rdparty/jvm/org/scalatestplus:junit", + "3rdparty/jvm/org/scalatestplus:mockito-1-10", "finagle/finagle-base-http/src/main", "finagle/finagle-core/src/main", "finagle/finagle-http/src/main/scala", diff --git a/slf4j-log4j12/src/test/scala/BUILD b/slf4j-log4j12/src/test/scala/BUILD index 25405f48..6ce980ff 100644 --- a/slf4j-log4j12/src/test/scala/BUILD +++ b/slf4j-log4j12/src/test/scala/BUILD @@ -5,6 +5,7 @@ junit_tests( "3rdparty/jvm/junit", "3rdparty/jvm/log4j", "3rdparty/jvm/org/scalatest", + "3rdparty/jvm/org/scalatestplus:junit", "3rdparty/jvm/org/slf4j:slf4j-api", "finagle/finagle-core/src/main", "twitter-server/server/src/main/scala", From 6334b66d8379ba090bef2cc521fb9ba2f2d901a1 Mon Sep 17 00:00:00 2001 From: Bonnie Eisenman Date: Fri, 17 Jul 2020 15:45:53 +0000 Subject: [PATCH 183/409] finagle-toggle: Change Toggle.isDefinedAt to Toggle.isDefined Problem ---------- Now that Toggle extends Function1 instead of PartialFunction, a toggle should always be either defined or undefned over the entire domain of `Int`. Solution ---------- Change `isDefinedAt(i: Int)` to `isDefined`, and update associated tests. JIRA Issues: CSL-9631 Differential Revision: https://phabricator.twitter.biz/D516868 --- .../com/twitter/server/handler/ToggleHandlerTest.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/src/test/scala/com/twitter/server/handler/ToggleHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/ToggleHandlerTest.scala index e1121ab8..eaae6f0f 100644 --- a/server/src/test/scala/com/twitter/server/handler/ToggleHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/ToggleHandlerTest.scala @@ -159,18 +159,18 @@ class ToggleHandlerTest extends FunSuite { val mut = ToggleMap.newMutable() val tog = mut(id) val handler = new ToggleHandler(() => Map(libName -> mut)) - assert(!tog.isDefinedAt(30)) + assert(tog.isUndefined) // mutate it val errors = handler.setToggle(libName, id, Some("1.0")) assert(errors.isEmpty, errors.mkString(", ")) - assert(tog.isDefinedAt(30)) + assert(tog.isDefined) assert(tog(30)) // mutate it again. val errors2 = handler.setToggle(libName, id, Some("0.0")) assert(errors2.isEmpty, errors.mkString(", ")) - assert(tog.isDefinedAt(30)) + assert(tog.isDefined) assert(!tog(30)) } @@ -209,13 +209,13 @@ class ToggleHandlerTest extends FunSuite { mut.put(id, 1.0) val tog = mut(id) val handler = new ToggleHandler(() => Map(libName -> mut)) - assert(tog.isDefinedAt(30)) + assert(tog.isDefined) assert(tog(30)) // delete it val errors = handler.deleteToggle(libName, id) assert(errors.isEmpty, errors.mkString(", ")) - assert(!tog.isDefinedAt(30)) + assert(tog.isUndefined) } test("parsePath") { From e991de92cee2372dfa50ee16c4169c650d2ebd62 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Fri, 17 Jul 2020 18:04:22 +0000 Subject: [PATCH 184/409] 3rdparty: Actually upgrade scalatest to 3.1.2 Problem In the previous commit where I claimed to have upgraded to 3.1.2, I actually upgraded to 3.1.1. I also broke the sbt build :grimacing: Solution Let's actually move to 3.1.2. We aren't moving to 3.2.0 yet because of the breaking API changes related to suite names. JIRA Issues: CSL-9722 Differential Revision: https://phabricator.twitter.biz/D518794 --- build.sbt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build.sbt b/build.sbt index 5f6e0307..bf85d22b 100644 --- a/build.sbt +++ b/build.sbt @@ -84,9 +84,9 @@ lazy val sharedSettings = Seq( "com.novocode" % "junit-interface" % "0.11" % "test", "org.mockito" % "mockito-all" % "1.9.5" % "test", "org.scalacheck" %% "scalacheck" % "1.14.3" % "test", - "org.scalatest" %% "scalatest" % "3.1.1" % "test" - "org.scalatestplus" %% "junit" % "3.1.2.0" % "test" - "org.scalatestplus" %% "mockito-1-10" % "3.1.1.0" % "test", + "org.scalatest" %% "scalatest" % "3.1.2" % "test", + "org.scalatestplus" %% "junit-4-12" % "3.1.2.0" % "test", + "org.scalatestplus" %% "mockito-1-10" % "3.1.0.0" % "test", "org.scalatestplus" %% "scalacheck-1-14" % "3.1.2.0" % "test" ), ScoverageKeys.coverageHighlighting := true, From 47a9943f02c7a205db8e2321f5ca8b19cff8f473 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Tue, 21 Jul 2020 18:07:20 +0000 Subject: [PATCH 185/409] twitter-oss: Prepare OSS libraries for release 20.7.0 Problem We want to release the next versions of our Twitter OSS libraries 20.7.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-9977 Differential Revision: https://phabricator.twitter.biz/D519631 --- CHANGELOG.rst | 3 +++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a55f4f5e..1a4abe44 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +20.7.0 +------ + Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ diff --git a/README.md b/README.md index 0735b712..88e9fbeb 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.com/twitter/twitter-server.svg?branch=develop)](https://travis-ci.com/twitter/twitter-server) +[![Build status](https://travis-ci.com/twitter/twitter-server.svg?branch=master)](https://travis-ci.com/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index bf85d22b..859dc80e 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.7.0-SNAPSHOT" +val releaseVersion = "20.7.0" val jacksonVersion = "2.11.0" val jacksonLibs = Seq( From 5cfc1fbe155fa75028d84a2eee5908a7917686bc Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Tue, 21 Jul 2020 19:14:43 +0000 Subject: [PATCH 186/409] twitter-oss: Update OSS Libraries Post-Release to 20.8.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 20.8.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. JIRA Issues: CSL-9977 Differential Revision: https://phabricator.twitter.biz/D519672 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 88e9fbeb..0735b712 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.com/twitter/twitter-server.svg?branch=master)](https://travis-ci.com/twitter/twitter-server) +[![Build status](https://travis-ci.com/twitter/twitter-server.svg?branch=develop)](https://travis-ci.com/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 859dc80e..7994f661 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.7.0" +val releaseVersion = "20.8.0-SNAPSHOT" val jacksonVersion = "2.11.0" val jacksonLibs = Seq( From 994e81e83fc9628473c06fd1dc04e868655ea88c Mon Sep 17 00:00:00 2001 From: Ruben Oanta Date: Thu, 23 Jul 2020 17:34:45 +0000 Subject: [PATCH 187/409] csl-libs: move master refs to release Problem / Solution We'd like to remove references to master in our OSS repos and replace them with release as it's more inclusive (and more accurately represents what this branch tracks). Differential Revision: https://phabricator.twitter.biz/D521022 --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0735b712..3fac126b 100644 --- a/README.md +++ b/README.md @@ -37,13 +37,13 @@ public API breaks and changes in runtime behavior. ## Contributing We feel that a welcoming community is important and we ask that you follow Twitter's -[Open Source Code of Conduct](https://github.com/twitter/code-of-conduct/blob/master/code-of-conduct.md) +[Open Source Code of Conduct](https://github.com/twitter/code-of-conduct/blob/release/code-of-conduct.md) in all interactions with the community. -The `master` branch of this repository contains the latest stable release of +The `release` branch of this repository contains the latest stable release of TwitterServer, and weekly snapshots are published to the `develop` branch. In general pull requests should be submitted against `develop`. See -[CONTRIBUTING.md](https://github.com/twitter/twitter-server/blob/master/CONTRIBUTING.md) +[CONTRIBUTING.md](https://github.com/twitter/twitter-server/blob/release/CONTRIBUTING.md) for more details about how to contribute. ## License From 34459e9c4759d3b0423f08f2cfa116b0b886396f Mon Sep 17 00:00:00 2001 From: Yi Cheng Date: Fri, 7 Aug 2020 19:49:08 +0000 Subject: [PATCH 188/409] [twitter-server] Remove 3rdparty/jvm/org/slf4j:slf4j-jdk14 from exports `3rdparty/jvm/org/slf4j:slf4j-jdk14` is a runtime only dep, so downstream compilation wouldn't need it. Differential Revision: https://phabricator.twitter.biz/D529560 --- slf4j-jdk14/src/main/scala/BUILD | 1 - 1 file changed, 1 deletion(-) diff --git a/slf4j-jdk14/src/main/scala/BUILD b/slf4j-jdk14/src/main/scala/BUILD index 86c83f9e..e1d3cd0b 100644 --- a/slf4j-jdk14/src/main/scala/BUILD +++ b/slf4j-jdk14/src/main/scala/BUILD @@ -21,7 +21,6 @@ scala_library( ], exports = [ "3rdparty/jvm/org/slf4j:slf4j-api", - "3rdparty/jvm/org/slf4j:slf4j-jdk14", "util/util-logging/src/main/scala", ], ) From 86d01bbef19bd1a8f2f0068d232b30d2fd0dd399 Mon Sep 17 00:00:00 2001 From: Marcy Reyes Date: Sat, 8 Aug 2020 00:14:07 +0000 Subject: [PATCH 189/409] Migrating Remaining OWNERS Files in source to PROJECT/PROJECT.yml We are finishing up our migration of OWNERS files in source to PROJECT files. In an effort to ensure PROJECT files are compatible with directories that already have project subdirectories, we are migrating ownership files in those particular directories to PROJECT.yml. For more information about PROJECT files, please see [[ https://docbird.twitter.biz/code-review/general/project_files.html | go/PROJECT ]]. JIRA Issues: CR-6197 Differential Revision: https://phabricator.twitter.biz/D528770 --- GROUPS | 1 - OWNERS | 7 ------- PROJECT.yml | 12 ++++++++++++ 3 files changed, 12 insertions(+), 8 deletions(-) delete mode 100644 GROUPS delete mode 100644 OWNERS create mode 100644 PROJECT.yml diff --git a/GROUPS b/GROUPS deleted file mode 100644 index e57fe3a5..00000000 --- a/GROUPS +++ /dev/null @@ -1 +0,0 @@ -csl diff --git a/OWNERS b/OWNERS deleted file mode 100644 index 995bde0b..00000000 --- a/OWNERS +++ /dev/null @@ -1,7 +0,0 @@ -banderson -ccoco -ibennett -koliver -mnakamura -roanta -ryano diff --git a/PROJECT.yml b/PROJECT.yml new file mode 100644 index 00000000..547f2b5d --- /dev/null +++ b/PROJECT.yml @@ -0,0 +1,12 @@ +# See go/project-docs. Generated using ./pants run src/python/twitter/phabricator:owners_to_project -- --dirs=a/owned/path,b/owned/path + +owners: + - banderson + - ccoco + - ibennett + - koliver + - mnakamura + - roanta + - ryano +watchers: + - csl@twitter.com From 3e64d5bbbd1c7d9aabd2dc14dbf062458824307c Mon Sep 17 00:00:00 2001 From: Ruben Oanta Date: Tue, 11 Aug 2020 02:53:48 +0000 Subject: [PATCH 190/409] Twitter-oss: Prepare OSS libraries for release 20.8.0 Problem We want to release the next versions of our Twitter OSS libraries 20.8.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D530623 --- CHANGELOG.rst | 5 +++++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1a4abe44..3fbd61e7 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +20.8.0 +------ + +No Changes + 20.7.0 ------ diff --git a/README.md b/README.md index 3fac126b..37015aec 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.com/twitter/twitter-server.svg?branch=develop)](https://travis-ci.com/twitter/twitter-server) +[![Build status](https://travis-ci.com/twitter/twitter-server.svg?branch=release)](https://travis-ci.com/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 7994f661..28806611 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.8.0-SNAPSHOT" +val releaseVersion = "20.8.0" val jacksonVersion = "2.11.0" val jacksonLibs = Seq( From 61df084611d8f4778e24a2ff5ddd836f35371239 Mon Sep 17 00:00:00 2001 From: Ruben Oanta Date: Tue, 11 Aug 2020 18:26:35 +0000 Subject: [PATCH 191/409] Twitter-oss: update OSS libraries post-release to 20.9.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 20.9.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D531226 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 37015aec..3fac126b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.com/twitter/twitter-server.svg?branch=release)](https://travis-ci.com/twitter/twitter-server) +[![Build status](https://travis-ci.com/twitter/twitter-server.svg?branch=develop)](https://travis-ci.com/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 28806611..feab9694 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.8.0" +val releaseVersion = "20.9.0-SNAPSHOT" val jacksonVersion = "2.11.0" val jacksonLibs = Seq( From ce783a3cadb9b27473006b59547adf22c534f5d9 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Wed, 12 Aug 2020 19:09:47 +0000 Subject: [PATCH 192/409] twitter-server: Add SecurityManager check to ContentionHandler Problem Attempting to access the `ManagementFactory.getThreadMXBean` to enable contention monitoring can throw a `SecurityException` if the `SecurityManager` does not allow for that permission. Solution Ensure that the `ContentionHandler` checks the `SecurityManager` before attempting to instantiate a `ContentionSnapshot`. JIRA Issues: CSL-10056 Differential Revision: https://phabricator.twitter.biz/D531873 --- CHANGELOG.rst | 3 + .../server/handler/ContentionHandler.scala | 57 +++++++++++---- .../handler/ContentionHandlerTest.scala | 72 +++++++++++++++++++ 3 files changed, 118 insertions(+), 14 deletions(-) create mode 100644 server/src/test/scala/com/twitter/server/handler/ContentionHandlerTest.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3fbd61e7..50f06fd1 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +* Check SecurityManager permissions in the `ContentHandler` to ensure that contention + snapshotting is allowed. ``PHAB_ID=D531873`` + 20.8.0 ------ diff --git a/server/src/main/scala/com/twitter/server/handler/ContentionHandler.scala b/server/src/main/scala/com/twitter/server/handler/ContentionHandler.scala index f31461eb..bda17036 100644 --- a/server/src/main/scala/com/twitter/server/handler/ContentionHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/ContentionHandler.scala @@ -5,23 +5,52 @@ import com.twitter.finagle.http.{Request, Response} import com.twitter.jvm.ContentionSnapshot import com.twitter.server.util.HttpUtils.newOk import com.twitter.util.Future +import com.twitter.util.logging.Logging +import java.lang.management.ManagementPermission + +class ContentionHandler extends Service[Request, Response] with Logging { + + // Must check the installed java.lang.SecurityManager for java.lang.management.ManagementPermission("control") + // access. If no java.lang.SecurityManager, we assume the instantiation of the ContentionSnapshot will succeed. + private[this] val contentionSnapshot: Option[ContentionSnapshot] = { + val permission = new ManagementPermission("control") + Option(System.getSecurityManager) match { + case Some(securityManager) => + try { + securityManager.checkPermission(permission) + Some(new ContentionSnapshot) + } catch { + case e: SecurityException => // not allowed + warn("Contention snapshotting is not allowed by SecurityManager.", e) + None + } + case _ => // no security manager + Some(new ContentionSnapshot) + } + } -class ContentionHandler extends Service[Request, Response] { - private[this] val snapshotter = new ContentionSnapshot def apply(req: Request): Future[Response] = { - val snap = snapshotter.snap - val deadlockMsg = - if (snap.deadlocks.isEmpty) "" - else { - "DEADLOCKS:\n\n%s\n\n".format(snap.deadlocks.mkString("\n\n")) - } + contentionSnapshot match { + case Some(snapshot) => + val snap = snapshot.snap() + val deadlockMsg = + if (snap.deadlocks.isEmpty) "" + else { + "DEADLOCKS:\n\n%s\n\n".format(snap.deadlocks.mkString("\n\n")) + } - val msg = "%sBlocked:\n%s\n\nLock Owners:\n%s".format( - deadlockMsg, - snap.blockedThreads.mkString("\n"), - snap.lockOwners.mkString("\n") - ) + val msg = "%sBlocked:\n%s\n\nLock Owners:\n%s".format( + deadlockMsg, + snap.blockedThreads.mkString("\n"), + snap.lockOwners.mkString("\n") + ) - newOk(msg) + newOk(msg) + case _ => + val msg = + "Contention snapshotting is not enabled due to SecurityManager restrictions.\n" + + "Please ensure that the java.lang.management.ManagementPermission(\"control\") is allowed." + newOk(msg) + } } } diff --git a/server/src/test/scala/com/twitter/server/handler/ContentionHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/ContentionHandlerTest.scala new file mode 100644 index 00000000..ae979e17 --- /dev/null +++ b/server/src/test/scala/com/twitter/server/handler/ContentionHandlerTest.scala @@ -0,0 +1,72 @@ +package com.twitter.server.handler + +import com.twitter.conversions.DurationOps._ +import com.twitter.finagle.http.{Request, Status} +import com.twitter.util.{Await, Awaitable, Promise} +import java.util.concurrent.locks.ReentrantLock +import org.scalatest.concurrent.Eventually +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.time.{Millis, Seconds, Span} + +object ContentionHandlerTest { + class Philosopher { + val ready = new Promise[Unit] + private val lock = new ReentrantLock() + def dine(neighbor: Philosopher): Unit = { + lock.lockInterruptibly() + ready.setValue(()) + Await.ready(neighbor.ready) + neighbor.dine(this) + lock.unlock() + } + } +} + +class ContentionHandlerTest extends AnyFunSuite with Eventually { + import ContentionHandlerTest._ + + implicit override val patienceConfig: PatienceConfig = { + PatienceConfig(timeout = scaled(Span(15, Seconds)), interval = scaled(Span(5, Millis))) + } + + test("ContentionHandler#instantiation") { + new ContentionHandler + } + + test("ContentionHandler#detect deadlocks") { + val handler = new ContentionHandler + + val descartes = new Philosopher() + val plato = new Philosopher() + + val d = new Thread(new Runnable() { + def run(): Unit = { descartes.dine(plato) } + }) + d.start() + + val p = new Thread(new Runnable() { + def run(): Unit = { plato.dine(descartes) } + }) + p.start() + Await.all(descartes.ready, plato.ready) + + eventually { + assertResponseMessage(handler, "DEADLOCKS:") + } + d.interrupt() + p.interrupt() + p.join() + d.join() + assertResponseMessage(handler, "") + } + + private[this] def await[T](a: Awaitable[T]): T = Await.result(a, 2.seconds) + + private[this] def assertResponseMessage(handler: ContentionHandler, message: String): Unit = { + val request = Request("/") + val response = await(handler.apply(request)) + assert(response.status == Status.Ok) + val responseBody = response.contentString + assert(responseBody.startsWith(message)) + } +} From 144ba5f9c34bb264428b590e10caad61260205bc Mon Sep 17 00:00:00 2001 From: Marcy Reyes Date: Fri, 21 Aug 2020 16:47:57 +0000 Subject: [PATCH 193/409] Corrected CSL team in PROJECT Files This is a follow up to the PROJECT/PROJECT.yml migration in source (D528770). The csl team was incorrectly included as csl@twitter.com. Updated all PROJECT files containing the CSL team to reflect them as csl-team:ldap. Differential Revision: https://phabricator.twitter.biz/D536529 --- PROJECT.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PROJECT.yml b/PROJECT.yml index 547f2b5d..8ee3c888 100644 --- a/PROJECT.yml +++ b/PROJECT.yml @@ -9,4 +9,4 @@ owners: - roanta - ryano watchers: - - csl@twitter.com + - csl-team:ldap From 3e63c5665960e803c90e769db46eac518382fc05 Mon Sep 17 00:00:00 2001 From: Bonnie Eisenman Date: Tue, 25 Aug 2020 20:09:26 +0000 Subject: [PATCH 194/409] Twitter-oss: Prepare OSS librares for patch release 20.8.1 Problem: We want to release the next versions of our Twitter OSS libraries 20.8.1 - util - scrooge - finagle - twitter-server - finatra Solution: Prepare libraries for their next release. Differential Revision: https://phabricator.twitter.biz/D538583 --- CHANGELOG.rst | 3 +++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 50f06fd1..2e95bd91 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +20.8.1 +------ + * Check SecurityManager permissions in the `ContentHandler` to ensure that contention snapshotting is allowed. ``PHAB_ID=D531873`` diff --git a/README.md b/README.md index 3fac126b..37015aec 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.com/twitter/twitter-server.svg?branch=develop)](https://travis-ci.com/twitter/twitter-server) +[![Build status](https://travis-ci.com/twitter/twitter-server.svg?branch=release)](https://travis-ci.com/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index feab9694..33e5384c 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.9.0-SNAPSHOT" +val releaseVersion = "20.8.1" val jacksonVersion = "2.11.0" val jacksonLibs = Seq( From 18b8b5276ab9b6d6787fc7a6eb074ba41d20853d Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Tue, 25 Aug 2020 20:59:28 +0000 Subject: [PATCH 195/409] twitter-server: Add Client Certificate Expiry Problem / Solution When clients are using client certificates to connect to an instance of TwitterServer, it can be helpful to know when the client's certificate is set to expire. Let's add the information to the `Incoming Connections` page. JIRA Issues: CSL-10067 Differential Revision: https://phabricator.twitter.biz/D528982 --- CHANGELOG.rst | 4 ++++ .../server/handler/AttachedClientsHandler.scala | 16 +++++++++++++--- .../handler/AttachedClientsHandlerTest.scala | 13 ++++++++++++- 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 2e95bd91..83b40a6c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,10 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +* If a client connecting to an instance of TwitterServer is sending a client certificate, + its expiry date (i.e. `Not After`) is now included as part of the information listed. + ``PHAB_ID=D528982``. + 20.8.1 ------ diff --git a/server/src/main/scala/com/twitter/server/handler/AttachedClientsHandler.scala b/server/src/main/scala/com/twitter/server/handler/AttachedClientsHandler.scala index abc8f131..1de48ca5 100644 --- a/server/src/main/scala/com/twitter/server/handler/AttachedClientsHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/AttachedClientsHandler.scala @@ -1,6 +1,6 @@ package com.twitter.server.handler -import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.{JsonFormat, JsonInclude} import com.fasterxml.jackson.annotation.JsonInclude.Include import com.fasterxml.jackson.databind.PropertyNamingStrategy import com.fasterxml.jackson.databind.annotation.JsonNaming @@ -10,13 +10,17 @@ import com.twitter.finagle.server.ServerRegistry import com.twitter.server.util.JsonConverter import com.twitter.util.Future import java.net.SocketAddress +import java.security.cert.X509Certificate +import java.util.Date private[handler] object AttachedClientsHandler { case class ClientConnectionEntry(address: SocketAddress, ssl: Option[ClientSslInfo]) @JsonNaming(classOf[PropertyNamingStrategy.SnakeCaseStrategy]) - case class PeerCertInfo(commonName: String) + case class PeerCertInfo( + commonName: String, + @JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSZ", timezone = "UTC") expiry: Date) @JsonNaming(classOf[PropertyNamingStrategy.SnakeCaseStrategy]) @JsonInclude(value = Include.NON_ABSENT, content = Include.NON_ABSENT) @@ -29,6 +33,12 @@ private[handler] object AttachedClientsHandler { case class AttachedClientsConnectionInfo(servers: Seq[ServerConnectionInfo]) + private def certToPeerCertInfo(certificate: X509Certificate): PeerCertInfo = + PeerCertInfo( + certificate.getSubjectDN.getName, + certificate.getNotAfter + ) + private def render(serverRegistry: ServerRegistry): AttachedClientsConnectionInfo = { AttachedClientsConnectionInfo(serverRegistry.serverAddresses.flatMap { serverAddress => val connectionRegistry = serverRegistry.connectionRegistry(serverAddress) @@ -46,7 +56,7 @@ private[handler] object AttachedClientsHandler { sessionId = clientConnection.sslSessionInfo.sessionId, cipherSuite = clientConnection.sslSessionInfo.cipherSuite, peerCertificate = clientConnection.sslSessionInfo.peerCertificates.headOption - .map { peerCertificate => PeerCertInfo(peerCertificate.getSubjectDN.getName) } + .map(certToPeerCertInfo) )) } ) diff --git a/server/src/test/scala/com/twitter/server/handler/AttachedClientsHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/AttachedClientsHandlerTest.scala index 4840a810..c8860994 100644 --- a/server/src/test/scala/com/twitter/server/handler/AttachedClientsHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/AttachedClientsHandlerTest.scala @@ -11,6 +11,7 @@ import com.twitter.util.{Await, Future, Time} import java.net.{InetSocketAddress, SocketAddress} import java.security.Principal import java.security.cert.X509Certificate +import java.util.Date import javax.net.ssl.SSLSession import org.mockito.Mockito import org.scalatest.FunSuite @@ -28,7 +29,16 @@ object AttachedClientsHandlerTest { val peerCertificate = Mockito.mock(classOf[X509Certificate]) val remotePrincipal = Mockito.mock(classOf[Principal]) + // We purposely use a date in the past here to validate the formatting + // of the json returned by this endpoint. Because it has already happened, + // it is not subject to any shenanigans regarding changes in leap seconds, + // leap years, or any other changes that might affect time values in the future. + // When running in production though, this endpoint should (almost) never return + // a timestamp from the past. Doing so would mean that a connection was established + // and then some time after, the certificate expired. + val remoteDate = new Date(1512295640000L) // Sun Dec 03 02:07:20 PST 2017 Mockito.when(peerCertificate.getSubjectDN).thenReturn(remotePrincipal) + Mockito.when(peerCertificate.getNotAfter).thenReturn(remoteDate) Mockito.when(remotePrincipal.getName).thenReturn("remoteprincipal") registry @@ -92,7 +102,8 @@ class AttachedClientsHandlerTest extends FunSuite { | "session_id": "sessionid", | "cipher_suite": "cipher?sweeeeet!", | "peer_certificate": { - | "common_name": "remoteprincipal" + | "common_name": "remoteprincipal", + | "expiry": "2017-12-03T10:07:20.000+0000" | } | } | } From e165c1e349e8f112cce9a641138806067ec556df Mon Sep 17 00:00:00 2001 From: Bonnie Eisenman Date: Wed, 26 Aug 2020 14:29:14 +0000 Subject: [PATCH 196/409] Twitter-oss: update OSS libraries post-release to 20.9.0 Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 20.9.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D538818 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 37015aec..3fac126b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.com/twitter/twitter-server.svg?branch=release)](https://travis-ci.com/twitter/twitter-server) +[![Build status](https://travis-ci.com/twitter/twitter-server.svg?branch=develop)](https://travis-ci.com/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 33e5384c..feab9694 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.8.1" +val releaseVersion = "20.9.0-SNAPSHOT" val jacksonVersion = "2.11.0" val jacksonLibs = Seq( From 86992eabf5197bed91bb0ebd277f66b39094f89b Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Thu, 27 Aug 2020 22:00:21 +0000 Subject: [PATCH 197/409] 3rdparty: Bump the jackson versions to 2.11.2 Problem We're on an old version of jackson Solution Upgrade!! JIRA Issues: CSL-10108 Differential Revision: https://phabricator.twitter.biz/D538440 --- CHANGELOG.rst | 2 ++ build.sbt | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 83b40a6c..6ae5728c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,8 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +* Bump version of Jackson to 2.11.2. ``PHAB_ID=D538440`` + * If a client connecting to an instance of TwitterServer is sending a client certificate, its expiry date (i.e. `Not After`) is now included as part of the information listed. ``PHAB_ID=D528982``. diff --git a/build.sbt b/build.sbt index feab9694..565d3f52 100644 --- a/build.sbt +++ b/build.sbt @@ -3,7 +3,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch val releaseVersion = "20.9.0-SNAPSHOT" -val jacksonVersion = "2.11.0" +val jacksonVersion = "2.11.2" val jacksonLibs = Seq( "com.fasterxml.jackson.core" % "jackson-core" % jacksonVersion, "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion, From 038ce6482d4b3bdd951dfb39abdd40039d87650a Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Fri, 28 Aug 2020 02:45:13 +0000 Subject: [PATCH 198/409] twitter-server: URL encode client/server labels Problem Client/Server labels can contain special characters that cause the request URL to fail. Solution Encode the URL and decode it when getting registry entry. JIRA Issues: CSL-8449 Differential Revision: https://phabricator.twitter.biz/D540543 --- CHANGELOG.rst | 3 +++ .../com/twitter/server/AdminHttpServer.scala | 9 +++++--- .../handler/ClientRegistryHandler.scala | 5 ++++- .../handler/ServerRegistryHandler.scala | 5 ++++- .../com/twitter/server/view/IndexView.scala | 2 +- .../handler/ClientRegistryHandlerTest.scala | 22 +++++++++++++++++++ .../handler/ServerRegistryHandlerTest.scala | 19 +++++++++++++++- 7 files changed, 58 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6ae5728c..901cb23b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,6 +9,9 @@ Unreleased * Bump version of Jackson to 2.11.2. ``PHAB_ID=D538440`` +* Encode the request URL names in /admin/clients/ and /admin/servers/. + ``PHAB_ID=D540543`` + * If a client connecting to an instance of TwitterServer is sending a client certificate, its expiry date (i.e. `Not After`) is now included as part of the information listed. ``PHAB_ID=D528982``. diff --git a/server/src/main/scala/com/twitter/server/AdminHttpServer.scala b/server/src/main/scala/com/twitter/server/AdminHttpServer.scala index 82c1e630..3b5b9a60 100644 --- a/server/src/main/scala/com/twitter/server/AdminHttpServer.scala +++ b/server/src/main/scala/com/twitter/server/AdminHttpServer.scala @@ -18,7 +18,8 @@ import com.twitter.server.view.{IndexView, NotFoundView} import com.twitter.util.lint.GlobalRules import com.twitter.util.registry.Library import com.twitter.util.{Future, Monitor, Time} -import java.net.InetSocketAddress +import java.net.{InetSocketAddress, URLEncoder} +import java.nio.charset.StandardCharsets import org.slf4j.LoggerFactory import scala.language.reflectiveCalls @@ -243,7 +244,8 @@ trait AdminHttpServer { self: App with Stats => private[this] def listeningServers: IndexView.Group = { val serverLinks: Seq[IndexView.Entry] = ServerRegistry.registrants.collect { case server if server.name.nonEmpty => - IndexView.Link(server.name, "/admin/servers/" + server.name) + val encodedName = URLEncoder.encode(server.name, StandardCharsets.UTF_8.name) + IndexView.Link(server.name, "/admin/servers/" + encodedName) }.toSeq IndexView.Group("Listening Servers", serverLinks.sorted(IndexView.EntryOrdering)) @@ -253,7 +255,8 @@ trait AdminHttpServer { self: App with Stats => private[this] def downstreamClients: IndexView.Group = { val clientLinks: Seq[IndexView.Entry] = ClientRegistry.registrants.collect { case client if client.name.nonEmpty => - IndexView.Link(client.name, "/admin/clients/" + client.name) + val encodedName = URLEncoder.encode(client.name, StandardCharsets.UTF_8.name) + IndexView.Link(client.name, "/admin/clients/" + encodedName) }.toSeq IndexView.Group("Downstream Clients", clientLinks.sorted(IndexView.EntryOrdering)) diff --git a/server/src/main/scala/com/twitter/server/handler/ClientRegistryHandler.scala b/server/src/main/scala/com/twitter/server/handler/ClientRegistryHandler.scala index cf353ade..5c7bc1f0 100644 --- a/server/src/main/scala/com/twitter/server/handler/ClientRegistryHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/ClientRegistryHandler.scala @@ -11,6 +11,8 @@ import com.twitter.server.util.HttpUtils.{new404, newResponse} import com.twitter.server.util.MetricSource import com.twitter.server.view.{BalancerHtmlView, EndpointRegistryView, StackRegistryView} import com.twitter.util.Future +import java.net.URLDecoder +import java.nio.charset.StandardCharsets private object ClientRegistryHandler { @@ -139,7 +141,8 @@ class ClientRegistryHandler( ) case name => - val clientEntries = stackRegistry.registrants.filter(_.name == name) + val decodedName = URLDecoder.decode(name, StandardCharsets.UTF_8.name) + val clientEntries = stackRegistry.registrants.filter(_.name == decodedName) if (clientEntries.isEmpty) new404(s"$name could not be found.") else { val client = clientEntries.head diff --git a/server/src/main/scala/com/twitter/server/handler/ServerRegistryHandler.scala b/server/src/main/scala/com/twitter/server/handler/ServerRegistryHandler.scala index 00b004fb..00040d76 100644 --- a/server/src/main/scala/com/twitter/server/handler/ServerRegistryHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/ServerRegistryHandler.scala @@ -10,6 +10,8 @@ import com.twitter.server.util.HttpUtils.{new404, newResponse} import com.twitter.server.util.MetricSource import com.twitter.server.view.StackRegistryView import com.twitter.util.Future +import java.net.URLDecoder +import java.nio.charset.StandardCharsets private object ServerRegistryHandler { def render(servers: Seq[(String, StackRegistry.Entry)]): String = @@ -96,7 +98,8 @@ class ServerRegistryHandler( ) case name => - val entries = registry.registrants filter { _.name == name } + val decodedName = URLDecoder.decode(name, StandardCharsets.UTF_8.name) + val entries = registry.registrants filter { _.name == decodedName } if (entries.isEmpty) new404(s"$name could not be found.") else { val server = entries.head diff --git a/server/src/main/scala/com/twitter/server/view/IndexView.scala b/server/src/main/scala/com/twitter/server/view/IndexView.scala index dd6d921b..5502b26b 100644 --- a/server/src/main/scala/com/twitter/server/view/IndexView.scala +++ b/server/src/main/scala/com/twitter/server/view/IndexView.scala @@ -5,7 +5,7 @@ import com.twitter.finagle.http.Method import com.twitter.finagle.http.Method.{Get, Post} import com.twitter.finagle.{Service, SimpleFilter} import com.twitter.finagle.http.{Request, Response} -import com.twitter.io.{Buf, BufReader, Reader, Pipe} +import com.twitter.io.{Buf, BufReader, Pipe, Reader} import com.twitter.server.util.HtmlUtils.escapeHtml import com.twitter.server.util.HttpUtils.{expectsHtml, newResponse} import com.twitter.util.Future diff --git a/server/src/test/scala/com/twitter/server/handler/ClientRegistryHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/ClientRegistryHandlerTest.scala index 965aa28b..958ca79f 100644 --- a/server/src/test/scala/com/twitter/server/handler/ClientRegistryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/ClientRegistryHandlerTest.scala @@ -7,6 +7,8 @@ import com.twitter.finagle.{Stack, param} import com.twitter.finagle.http.{Request, Status} import com.twitter.server.util.MetricSourceTest import com.twitter.util.{Await, Awaitable, Time} +import java.net.URLEncoder +import java.nio.charset.StandardCharsets import org.scalatest.FunSuite class ClientRegistryHandlerTest extends FunSuite { @@ -81,4 +83,24 @@ class ClientRegistryHandlerTest extends FunSuite { assert(html.contains("98.00%")) } } + + test("client name is decoded in handler") { + val metricsCtx = new MetricSourceTest.Ctx + import metricsCtx._ + + val registry = new StackRegistry { + def registryName: String = "client" + } + val clientName = "/srv#/abc/abc" + registry.register( + "localhost:8080", + StackClient.newStack, + Stack.Params.empty + param.Label(clientName) + ) + val handler = new ClientRegistryHandler("/admin/clients/", source, registry) + + val req = + Request("/admin/clients/" + URLEncoder.encode(clientName, StandardCharsets.UTF_8.name)) + assert(await(handler(req)).status == Status.Ok) + } } diff --git a/server/src/test/scala/com/twitter/server/handler/ServerRegistryHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/ServerRegistryHandlerTest.scala index a1dac881..f45525e6 100644 --- a/server/src/test/scala/com/twitter/server/handler/ServerRegistryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/ServerRegistryHandlerTest.scala @@ -2,10 +2,12 @@ package com.twitter.server.handler import com.twitter.conversions.DurationOps._ import com.twitter.finagle.server.StackServer -import com.twitter.finagle.{http, Stack, param} +import com.twitter.finagle.{Stack, http, param} import com.twitter.finagle.util.StackRegistry import com.twitter.server.util.MetricSourceTest import com.twitter.util.{Await, Awaitable} +import java.net.URLEncoder +import java.nio.charset.StandardCharsets import org.scalatest.FunSuite class ServerRegistryHandlerTest extends FunSuite { @@ -49,4 +51,19 @@ class ServerRegistryHandlerTest extends FunSuite { val res = await(handler(http.Request("/admin/servers/index.html"))) assert(res.status == http.Status.Ok) } + + test("server name is decoded in handler") { + val metricsCtx = new MetricSourceTest.Ctx + import metricsCtx._ + + val registry = new StackRegistry { def registryName: String = "server" } + val serverName = "server@1" + registry.register(":8080", StackServer.newStack, Stack.Params.empty + param.Label(serverName)) + + val handler = new ServerRegistryHandler("/admin/servers/", source, registry) + + val res = await(handler( + http.Request("/admin/servers/" + URLEncoder.encode(serverName, StandardCharsets.UTF_8.name)))) + assert(res.status == http.Status.Ok) + } } From 2afbd1398bfd67d08996911881ff6bd49528993e Mon Sep 17 00:00:00 2001 From: Ekaterina Tyurina Date: Wed, 2 Sep 2020 04:06:52 +0000 Subject: [PATCH 199/409] [csl targets] explicitly list out the runtime scoped dependencies **Problem and Solution** Pants can specify the runtime scope at dep level so it is implicit at the dependee level, whereas bazel cannot, so this change makes the runtime deps explicit for bazel to understand. JIRA Issues: DPB-12421 Differential Revision: https://phabricator.twitter.biz/D529617 --- logback-classic/src/main/scala/BUILD | 10 ++++++++-- server/src/test/java/BUILD | 5 ++++- server/src/test/scala/BUILD | 5 ++++- slf4j-jdk14/src/main/scala/BUILD | 10 ++++++++-- slf4j-log4j12/src/main/scala/BUILD | 10 ++++++++-- 5 files changed, 32 insertions(+), 8 deletions(-) diff --git a/logback-classic/src/main/scala/BUILD b/logback-classic/src/main/scala/BUILD index d706bba7..586236dc 100644 --- a/logback-classic/src/main/scala/BUILD +++ b/logback-classic/src/main/scala/BUILD @@ -11,10 +11,16 @@ scala_library( dependencies = [ "3rdparty/jvm/ch/qos/logback:logback-classic", "3rdparty/jvm/ch/qos/logback:logback-core", - "3rdparty/jvm/org/slf4j:jcl-over-slf4j", "3rdparty/jvm/org/slf4j:jul-to-slf4j", - "3rdparty/jvm/org/slf4j:log4j-over-slf4j", "3rdparty/jvm/org/slf4j:slf4j-api", + scoped( + "3rdparty/jvm/org/slf4j:jcl-over-slf4j", + scope = "runtime", + ), + scoped( + "3rdparty/jvm/org/slf4j:log4j-over-slf4j", + scope = "runtime", + ), "finagle/finagle-base-http/src/main", "finagle/finagle-http/src/main/scala", "twitter-server/logback-classic/src/main/resources", diff --git a/server/src/test/java/BUILD b/server/src/test/java/BUILD index 47b5ea2c..ce7af35a 100644 --- a/server/src/test/java/BUILD +++ b/server/src/test/java/BUILD @@ -9,7 +9,10 @@ junit_tests( "3rdparty/jvm/org/scalatestplus:junit", "3rdparty/jvm/org/scalatestplus:mockito-1-10", "3rdparty/jvm/org/slf4j:slf4j-api", - "3rdparty/jvm/org/slf4j:slf4j-nop", + scoped( + "3rdparty/jvm/org/slf4j:slf4j-nop", + scope = "runtime", + ), "finagle/finagle-core/src/main", "finagle/finagle-http/src/main/scala", "twitter-server/server/src/main/scala", diff --git a/server/src/test/scala/BUILD b/server/src/test/scala/BUILD index 71ff0fc9..7de6e580 100644 --- a/server/src/test/scala/BUILD +++ b/server/src/test/scala/BUILD @@ -12,7 +12,10 @@ junit_tests( "3rdparty/jvm/org/scalatestplus:mockito-1-10", "3rdparty/jvm/org/scalatestplus:scalacheck", "3rdparty/jvm/org/slf4j:slf4j-api", - "3rdparty/jvm/org/slf4j:slf4j-nop", + scoped( + "3rdparty/jvm/org/slf4j:slf4j-nop", + scope = "runtime", + ), "finagle/finagle-core/src/main", "finagle/finagle-http/src/main/scala", "finagle/finagle-toggle", diff --git a/slf4j-jdk14/src/main/scala/BUILD b/slf4j-jdk14/src/main/scala/BUILD index e1d3cd0b..d3715b30 100644 --- a/slf4j-jdk14/src/main/scala/BUILD +++ b/slf4j-jdk14/src/main/scala/BUILD @@ -9,9 +9,15 @@ scala_library( scope = "forced", # Exposes a service-loaded class. tags = ["logging_impl_check_whitelisted_target"], dependencies = [ - "3rdparty/jvm/org/slf4j:jcl-over-slf4j", - "3rdparty/jvm/org/slf4j:log4j-over-slf4j", "3rdparty/jvm/org/slf4j:slf4j-api", + scoped( + "3rdparty/jvm/org/slf4j:jcl-over-slf4j", + scope = "runtime", + ), + scoped( + "3rdparty/jvm/org/slf4j:log4j-over-slf4j", + scope = "runtime", + ), "3rdparty/jvm/org/slf4j:slf4j-jdk14", "finagle/finagle-base-http/src/main", "finagle/finagle-http/src/main/scala", diff --git a/slf4j-log4j12/src/main/scala/BUILD b/slf4j-log4j12/src/main/scala/BUILD index 1de45115..778bc32f 100644 --- a/slf4j-log4j12/src/main/scala/BUILD +++ b/slf4j-log4j12/src/main/scala/BUILD @@ -10,10 +10,16 @@ scala_library( tags = ["logging_impl_check_whitelisted_target"], dependencies = [ "3rdparty/jvm/log4j", - "3rdparty/jvm/org/slf4j:jcl-over-slf4j", "3rdparty/jvm/org/slf4j:jul-to-slf4j", "3rdparty/jvm/org/slf4j:slf4j-api", - "3rdparty/jvm/org/slf4j:slf4j-log4j12", + scoped( + "3rdparty/jvm/org/slf4j:jcl-over-slf4j", + scope = "runtime", + ), + scoped( + "3rdparty/jvm/org/slf4j:slf4j-log4j12", + scope = "runtime", + ), "finagle/finagle-base-http/src/main", "finagle/finagle-http/src/main/scala", "twitter-server/server/src/main/scala", From d821ceff262f92ea893187da3f515ab6e46bfc72 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Fri, 4 Sep 2020 16:16:23 +0000 Subject: [PATCH 200/409] twitter-server: Remove allow failures for Scala 2.13 Problem/Solution We no longer need to specify allow failures for Scala 2.13 in the TwitterServer TravisCI configuration since the dependency chain is fully 2.13 compatible now. Let's remove. Differential Revision: https://phabricator.twitter.biz/D544940 --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index cec2ac12..6f8b7c00 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,8 +36,6 @@ matrix: # Does not allow for assigning a build result if the non-'allow_failures' targets are complete # because of https://travis-ci.community/t/receiving-too-many-emails/4888 fast_finish: false - allow_failures: - - scala: 2.13.1 notifications: slack: From c844072059b31eafddc65e9ea4e2038c2a4daa83 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Thu, 10 Sep 2020 17:53:33 +0000 Subject: [PATCH 201/409] scrooge|finagle|twitter-server|finatra: Let's talk about snapshots Problem/Solution Update CONTRIBUTING documentation to mention published nightly snapshots. Differential Revision: https://phabricator.twitter.biz/D547194 --- CONTRIBUTING.md | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5c3b0802..06cc652b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,17 +2,28 @@ We'd love to get patches from you! +## Nightly Snapshots + +Snapshots are published nightly for the current version in development and are +available in the [Sonatype](https://oss.sonatype.org/) open source snapshot +repository: [https://oss.sonatype.org/content/repositories/snapshots/](https://oss.sonatype.org/content/repositories/snapshots/). + ## Building dependencies -We are not currently publishing snapshots for TwitterServer's dependencies, which -means that it may be necessary to publish the `develop` branches of these -libraries locally in order to work on TwitterServer's `develop` branch. To do so -you can use our build tool, [dodo](https://github.com/twitter/dodo). +If you want to manually build and publish the `develop` branches of TwitterServer's +dependencies locally, you can use our build tool, [dodo](https://github.com/twitter/dodo). ``` bash curl -s https://raw.githubusercontent.com/twitter/dodo/develop/bin/build | bash -s -- --no-test twitter-server ``` +This will clone, build, and publish locally the current `-SNAPSHOT` version +from the `develop` branch of TwitterServer's other Twitter open source dependencies. + +It is your choice to use the published nightly snapshot versions or to build +the snapshots locally from their respective `develop` branches via the +[dodo](https://github.com/twitter/dodo) build tool. + ## Building TwitterServer TwitterServer is built using [sbt][sbt]. When building please use the included @@ -28,7 +39,7 @@ the Finaglers [mailing list](https://groups.google.com/forum/#!forum/finaglers). The workflow that we support: -1. Fork finagle +1. Fork twitter-server 1. Check out the `develop` branch 1. Make a feature branch (use `git checkout -b "cool-new-feature"`) 1. Make your cool new feature or bugfix on your branch From 455e422ab9441cc975b62934fe0ed9f918561338 Mon Sep 17 00:00:00 2001 From: yic Date: Mon, 14 Sep 2020 23:04:40 +0000 Subject: [PATCH 202/409] [twitter-server] change test target type JIRA Issues: DPB-12589 Differential Revision: https://phabricator.twitter.biz/D549355 --- server/src/test/java/BUILD | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/BUILD b/server/src/test/java/BUILD index ce7af35a..27a0b809 100644 --- a/server/src/test/java/BUILD +++ b/server/src/test/java/BUILD @@ -1,4 +1,6 @@ -junit_tests( +# The compilation of this target acts as a test to make sure scala APIs work in Java. +# There is no test within this target. +java_library( sources = ["**/*.java"], compiler_option_sets = [], dependencies = [ From 93a21193fea6ad2a3d3de25b2a6464b38e0c0b5c Mon Sep 17 00:00:00 2001 From: Yi Cheng Date: Tue, 15 Sep 2020 16:41:29 +0000 Subject: [PATCH 203/409] [twitter-server] Separate out compile and runtime dep JIRA Issues: DPB-12589 Differential Revision: https://phabricator.twitter.biz/D549333 --- slf4j-jdk14/src/test/scala/BUILD | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/slf4j-jdk14/src/test/scala/BUILD b/slf4j-jdk14/src/test/scala/BUILD index 0d4289b0..3a585299 100644 --- a/slf4j-jdk14/src/test/scala/BUILD +++ b/slf4j-jdk14/src/test/scala/BUILD @@ -8,10 +8,14 @@ junit_tests( "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", "3rdparty/jvm/org/scalatestplus:mockito-1-10", + scoped( + "twitter-server/slf4j-jdk14/src/main/scala", + scope = "runtime", + ), "finagle/finagle-base-http/src/main", "finagle/finagle-core/src/main", "finagle/finagle-http/src/main/scala", "twitter-server/server/src/main/scala", - "twitter-server/slf4j-jdk14/src/main/scala", + "twitter-server/slf4j-jdk14/src/main/scala/com/twitter/server/logging", ], ) From f0f211e77a0c136f0839acaf29cf1bef15a48395 Mon Sep 17 00:00:00 2001 From: Yi Cheng Date: Wed, 16 Sep 2020 04:23:39 +0000 Subject: [PATCH 204/409] [bazel] mark twitter-server as testable by bazel JIRA Issues: DPB-12589 Differential Revision: https://phabricator.twitter.biz/D550303 --- slf4j-jdk14/src/test/scala/BUILD | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/slf4j-jdk14/src/test/scala/BUILD b/slf4j-jdk14/src/test/scala/BUILD index 3a585299..d1927384 100644 --- a/slf4j-jdk14/src/test/scala/BUILD +++ b/slf4j-jdk14/src/test/scala/BUILD @@ -12,6 +12,10 @@ junit_tests( "twitter-server/slf4j-jdk14/src/main/scala", scope = "runtime", ), + scoped( + "twitter-server/slf4j-jdk14/src/main/scala/com/twitter/server/handler/slf4j/jdk14", + scope = "runtime" + ), "finagle/finagle-base-http/src/main", "finagle/finagle-core/src/main", "finagle/finagle-http/src/main/scala", From 05880cea7a2b4503207fc45256784af151abcc89 Mon Sep 17 00:00:00 2001 From: Hamdi Allam Date: Tue, 22 Sep 2020 20:21:39 +0000 Subject: [PATCH 205/409] Twitter-oss: Prepare OSS libraries for release 20.9.0 Problem We want to release the next versions of our Twitter OSS libraries 20.9.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-10167 Differential Revision: https://phabricator.twitter.biz/D553327 --- CHANGELOG.rst | 5 +++++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 901cb23b..4dee64db 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +20.9.0 +------ + +No Changes + * Bump version of Jackson to 2.11.2. ``PHAB_ID=D538440`` * Encode the request URL names in /admin/clients/ and /admin/servers/. diff --git a/README.md b/README.md index 3fac126b..37015aec 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.com/twitter/twitter-server.svg?branch=develop)](https://travis-ci.com/twitter/twitter-server) +[![Build status](https://travis-ci.com/twitter/twitter-server.svg?branch=release)](https://travis-ci.com/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 565d3f52..acea3be4 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.9.0-SNAPSHOT" +val releaseVersion = "20.9.0" val jacksonVersion = "2.11.2" val jacksonLibs = Seq( From 944a6bc73bd4bae18041fdc8467c223425eea64b Mon Sep 17 00:00:00 2001 From: Hamdi Allam Date: Tue, 22 Sep 2020 23:02:54 +0000 Subject: [PATCH 206/409] Twitter-oss: update OSS libraries post-release to 20.10.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 20.10.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. JIRA Issues: CSL-10167 Differential Revision: https://phabricator.twitter.biz/D553399 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index acea3be4..f157778e 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.9.0" +val releaseVersion = "20.10.0-SNAPSHOT" val jacksonVersion = "2.11.2" val jacksonLibs = Seq( From 8b49adea5ff2ddd05f45c4565877679f1b21d79f Mon Sep 17 00:00:00 2001 From: Matt Dannenberg Date: Thu, 24 Sep 2020 17:39:35 +0000 Subject: [PATCH 207/409] finagle-core: set relative_name metadata for finagle client and server metrics Problem ------- Users currently filter metrics via regexp on the metric name. This can lead to misidentifying metrics (for example, when looking for failures metrics a client may end up with both the failures metrics and the tries/failures metric, which includes retries). Solution -------- Add a relative_name field to metrics metadata which the users can filter on directly. This is enabled by a new StatsReceiver which marks the scope beyond which the relative metric name can be found. Result ------ Client and Server related metrics created by finagle now have their relative_names configured and exposed via metrics metadata. JIRA Issues: CSL-10202 Differential Revision: https://phabricator.twitter.biz/D552357 --- CHANGELOG.rst | 6 +++++ .../handler/MetricMetadataQueryHandler.scala | 7 ++++-- .../server/util/MetricSchemaJsonModule.scala | 7 ++++++ .../MetricMetadataQueryHandlerTest.scala | 24 +++++++++++++------ .../util/MetricSchemaJsonModuleTest.scala | 10 +++++++- 5 files changed, 44 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4dee64db..ffa1863c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -21,6 +21,12 @@ No Changes its expiry date (i.e. `Not After`) is now included as part of the information listed. ``PHAB_ID=D528982``. +Breaking API Changes +~~~~~~~~~~~~~~~~~~~~ + +* Add relative_name field to metrics in the Metrics Metadata endpoint and bump the + endpoints version number to 2.0. ``PHAB_ID=D552357`` + 20.8.1 ------ diff --git a/server/src/main/scala/com/twitter/server/handler/MetricMetadataQueryHandler.scala b/server/src/main/scala/com/twitter/server/handler/MetricMetadataQueryHandler.scala index 0f358332..76907717 100644 --- a/server/src/main/scala/com/twitter/server/handler/MetricMetadataQueryHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/MetricMetadataQueryHandler.scala @@ -20,11 +20,12 @@ import com.twitter.util.Future * * Response: * { - * "@version" : 1.0, + * "@version" : 2.0, * "counters_latched" : false, * "metrics" : [ * { * "name" : "my/cool/counter", + * "relative_name" : ["cool","counter"], * "kind" : "counter", * "source" : { * "class": "finagle.stats.cool", @@ -38,6 +39,7 @@ import com.twitter.util.Future * }, * { * "name" : "your/fine/gauge", + * "relative_name" : ["fine","gauge"], * "kind" : "gauge", * "source" : { * "class": "finagle.stats.your", @@ -51,6 +53,7 @@ import com.twitter.util.Future * }, * { * "name" : "my/only/histo", + * "relative_name" : ["histo"], * "kind" : "histogram", * "source" : { * "class": "Unspecified", @@ -92,7 +95,7 @@ class MetricMetadataQueryHandler(source: MetricSchemaSource = new MetricSchemaSo content = Buf.Utf8( JsonConverter.writeToString( Map( - "@version" -> 1.0, + "@version" -> 2.0, "counters_latched" -> latched, "metrics" -> metrics ))) diff --git a/server/src/main/scala/com/twitter/server/util/MetricSchemaJsonModule.scala b/server/src/main/scala/com/twitter/server/util/MetricSchemaJsonModule.scala index 50ce5cf0..a7c324d3 100644 --- a/server/src/main/scala/com/twitter/server/util/MetricSchemaJsonModule.scala +++ b/server/src/main/scala/com/twitter/server/util/MetricSchemaJsonModule.scala @@ -27,6 +27,13 @@ object SchemaSerializer extends StdSerializer[MetricSchema](classOf[MetricSchema ): Unit = { jsonGenerator.writeStartObject() jsonGenerator.writeStringField("name", metricSchema.metricBuilder.name.mkString("/")) + jsonGenerator.writeArrayFieldStart("relative_name") + if (metricSchema.metricBuilder.relativeName != Seq.empty) { + metricSchema.metricBuilder.relativeName.foreach(segment => jsonGenerator.writeString(segment)) + } else { + metricSchema.metricBuilder.name.foreach(segment => jsonGenerator.writeString(segment)) + } + jsonGenerator.writeEndArray() val dataType = metricSchema match { case _: CounterSchema => "counter" case _: GaugeSchema => "gauge" diff --git a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala index 944e4fd1..722fe6a3 100644 --- a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala @@ -70,24 +70,26 @@ class MetricMetadataQueryHandlerTest extends FunSuite { private[this] val unlatchedHandler = new MetricMetadataQueryHandler(unlatchedMetricSchemaSource) - val typeRequestNoArg = Request("http://$HOST:$PORT/admin/metric_metadata") + val typeRequestNoArg = Request("http://$HOST:$PORT/admin/metric_metadata.json") val typeRequestWithAnArg = Request( - "http://$HOST:$PORT/admin/metric_metadata?name=your/fine/gauge") + "http://$HOST:$PORT/admin/metric_metadata.json?name=your/fine/gauge") val typeRequestWithManyArgs = Request( - "http://$HOST:$PORT/admin/metric_metadata?name=my/cool/counter&name=your/fine/gauge") + "http://$HOST:$PORT/admin/metric_metadata.json?name=my/cool/counter&name=your/fine/gauge") - val typeRequestWithHisto = Request("http://$HOST:$PORT/admin/metric_metadata?name=my/only/histo") + val typeRequestWithHisto = Request( + "http://$HOST:$PORT/admin/metric_metadata.json?name=my/only/histo") val typeRequestWithHistoAndNon = Request( - "http://$HOST:$PORT/admin/metric_metadata?name=my/cool/counter&name=my/only/histo") + "http://$HOST:$PORT/admin/metric_metadata.json?name=my/cool/counter&name=my/only/histo") val responseToNoArg = """ | "metrics" : [ | { | "name" : "my/cool/counter", + | "relative_name" : ["my","cool","counter"], | "kind" : "counter", | "source" : { | "class": "finagle.stats.cool", @@ -101,6 +103,7 @@ class MetricMetadataQueryHandlerTest extends FunSuite { | }, | { | "name" : "your/fine/gauge", + | "relative_name" : ["your","fine","gauge"], | "kind" : "gauge", | "source" : { | "class": "finagle.stats.your", @@ -114,6 +117,7 @@ class MetricMetadataQueryHandlerTest extends FunSuite { | }, | { | "name" : "my/only/histo", + | "relative_name" : ["my","only","histo"], | "kind" : "histogram", | "source" : { | "class": "Unspecified", @@ -142,6 +146,7 @@ class MetricMetadataQueryHandlerTest extends FunSuite { | "metrics" : [ | { | "name" : "your/fine/gauge", + | "relative_name" : ["your","fine","gauge"], | "kind" : "gauge", | "source" : { | "class": "finagle.stats.your", @@ -161,6 +166,7 @@ class MetricMetadataQueryHandlerTest extends FunSuite { | "metrics" : [ | { | "name" : "my/cool/counter", + | "relative_name" : ["my","cool","counter"], | "kind" : "counter", | "source" : { | "class": "finagle.stats.cool", @@ -174,6 +180,7 @@ class MetricMetadataQueryHandlerTest extends FunSuite { | }, | { | "name" : "your/fine/gauge", + | "relative_name" : ["your","fine","gauge"], | "kind" : "gauge", | "source" : { | "class": "finagle.stats.your", @@ -194,6 +201,7 @@ class MetricMetadataQueryHandlerTest extends FunSuite { | "metrics" : [ | { | "name" : "my/only/histo", + | "relative_name" : ["my","only","histo"], | "kind" : "histogram", | "source" : { | "class": "Unspecified", @@ -221,6 +229,7 @@ class MetricMetadataQueryHandlerTest extends FunSuite { | "metrics" : [ | { | "name" : "my/cool/counter", + | "relative_name" : ["my","cool","counter"], | "kind" : "counter", | "source" : { | "class": "finagle.stats.cool", @@ -234,6 +243,7 @@ class MetricMetadataQueryHandlerTest extends FunSuite { | }, | { | "name" : "my/only/histo", + | "relative_name" : ["my","only","histo"], | "kind" : "histogram", | "source" : { | "class": "Unspecified", @@ -297,7 +307,7 @@ class MetricMetadataQueryHandlerTest extends FunSuite { val responseStart = """ | { - | "@version" : 1.0, + | "@version" : 2.0, | "counters_latched" : true, """.stripMargin test(testName + " when using latched counters") { @@ -310,7 +320,7 @@ class MetricMetadataQueryHandlerTest extends FunSuite { val responseStart = """ | { - | "@version" : 1.0, + | "@version" : 2.0, | "counters_latched" : false, """.stripMargin test(testName + " when using unlatched counters") { diff --git a/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala b/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala index 7de7373c..25bd0e32 100644 --- a/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala +++ b/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala @@ -41,7 +41,15 @@ class MetricSchemaJsonModuleTest extends FunSuite { )) private val topLevelFieldNameSet = - Set("name", "kind", "source", "description", "unit", "verbosity", "key_indicator") + Set( + "name", + "relative_name", + "kind", + "source", + "description", + "unit", + "verbosity", + "key_indicator") def jsonStrToMap(jsonStr: String): Map[String, Any] = { val mapper = new ObjectMapper() with ScalaObjectMapper From 792c18eddf6437022161d1910f7aaa5e87572a26 Mon Sep 17 00:00:00 2001 From: Matt Dannenberg Date: Thu, 1 Oct 2020 21:56:53 +0000 Subject: [PATCH 208/409] util-stats: use service scopeSeparator in MetricBuilder Problem ------- Metric Metadata uses / as the scope separator in all cases. As a result, when service owners configure a custom scope separator (ex, -) the metric names will differ in metrics.json and metric_metadata.json. Solution -------- Create an object in util-stats to store a scope separating character for MetricBuilder and metric_metadata.json to make use of. Result ------ Metric names now align across metrics.json and metric_metadata.json for services with custom configured scope separators JIRA Issues: CSL-10202 Differential Revision: https://phabricator.twitter.biz/D557994 --- .../twitter/server/util/MetricSchemaJsonModule.scala | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/util/MetricSchemaJsonModule.scala b/server/src/main/scala/com/twitter/server/util/MetricSchemaJsonModule.scala index a7c324d3..6e38f202 100644 --- a/server/src/main/scala/com/twitter/server/util/MetricSchemaJsonModule.scala +++ b/server/src/main/scala/com/twitter/server/util/MetricSchemaJsonModule.scala @@ -4,7 +4,13 @@ import com.fasterxml.jackson.core.JsonGenerator import com.fasterxml.jackson.databind.SerializerProvider import com.fasterxml.jackson.databind.module.SimpleModule import com.fasterxml.jackson.databind.ser.std.StdSerializer -import com.twitter.finagle.stats.{CounterSchema, GaugeSchema, HistogramSchema, MetricSchema} +import com.twitter.finagle.stats.{ + CounterSchema, + GaugeSchema, + HistogramSchema, + MetricSchema, + metadataScopeSeparator +} object SchemaSerializer extends StdSerializer[MetricSchema](classOf[MetricSchema]) { @@ -26,7 +32,9 @@ object SchemaSerializer extends StdSerializer[MetricSchema](classOf[MetricSchema serializerProvider: SerializerProvider ): Unit = { jsonGenerator.writeStartObject() - jsonGenerator.writeStringField("name", metricSchema.metricBuilder.name.mkString("/")) + jsonGenerator.writeStringField( + "name", + metricSchema.metricBuilder.name.mkString(metadataScopeSeparator())) jsonGenerator.writeArrayFieldStart("relative_name") if (metricSchema.metricBuilder.relativeName != Seq.empty) { metricSchema.metricBuilder.relativeName.foreach(segment => jsonGenerator.writeString(segment)) From 5bd1fb6a8aca5f1e8b3fa14476357279be888037 Mon Sep 17 00:00:00 2001 From: Hamdi Allam Date: Tue, 27 Oct 2020 01:02:07 +0000 Subject: [PATCH 209/409] Twitter-oss: Prepare OSS libraries for release 20.10.0 Problem We want to release the next versions of our Twitter OSS libraries 20.10.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-10323 Differential Revision: https://phabricator.twitter.biz/D571081 --- CHANGELOG.rst | 5 +++++ build.sbt | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ffa1863c..df6b53c6 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +20.10.0 +------- + +No Changes + 20.9.0 ------ diff --git a/build.sbt b/build.sbt index f157778e..98d5ce50 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.10.0-SNAPSHOT" +val releaseVersion = "20.10.0" val jacksonVersion = "2.11.2" val jacksonLibs = Seq( From 3bddefcce2e960735ae1d62f9e88d43419f272c1 Mon Sep 17 00:00:00 2001 From: Hamdi Allam Date: Tue, 27 Oct 2020 15:18:54 +0000 Subject: [PATCH 210/409] Twitter-oss: update OSS libraries post-release to 20.11.0 Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 20.11.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. JIRA Issues: CSL-10323 Differential Revision: https://phabricator.twitter.biz/D571343 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 37015aec..3fac126b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.com/twitter/twitter-server.svg?branch=release)](https://travis-ci.com/twitter/twitter-server) +[![Build status](https://travis-ci.com/twitter/twitter-server.svg?branch=develop)](https://travis-ci.com/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 98d5ce50..bccf8551 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.10.0" +val releaseVersion = "20.11.0-SNAPSHOT" val jacksonVersion = "2.11.2" val jacksonLibs = Seq( From e0aeb87e89a6e6c711214ee2de0dd9f6e5f9cb6c Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Tue, 10 Nov 2020 16:14:44 +0000 Subject: [PATCH 211/409] twitter-server: Plug XSS vulnerability in /histograms endpoint Problem Right now we allow script injection in /histograms by telling people about the query that they just ran. Solution Stop telling people about the query they ran. JIRA Issues: OSS-5328 Differential Revision: https://phabricator.twitter.biz/D574844 --- CHANGELOG.rst | 3 +++ .../com/twitter/server/handler/HistogramQueryHandler.scala | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index df6b53c6..6ed485a1 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +* Escape user-provided string from the returned text. This removes a potential vector for an XSS + attack. ``PHAB_ID=D574844`` + 20.10.0 ------- diff --git a/server/src/main/scala/com/twitter/server/handler/HistogramQueryHandler.scala b/server/src/main/scala/com/twitter/server/handler/HistogramQueryHandler.scala index bf87e6e5..951c255a 100644 --- a/server/src/main/scala/com/twitter/server/handler/HistogramQueryHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/HistogramQueryHandler.scala @@ -198,7 +198,7 @@ private[server] class HistogramQueryHandler(details: WithHistogramDetails) content = { val text = histograms.get(query) match { case Some(h) => transform(h.counts) - case None => s"Key: $query is not a valid histogram." + case None => s"Key: ${escapeHtml(query)} is not a valid histogram." } Buf.Utf8(text) } @@ -280,7 +280,7 @@ private[server] class HistogramQueryHandler(details: WithHistogramDetails) if (histograms.contains(query)) render else - s"Key: $query is not a valid histogram." + s"Key: ${escapeHtml(query)} is not a valid histogram." } ) From 3c671b7ec5e0bed1adaba4d76ae752e583c16507 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Mon, 16 Nov 2020 20:21:29 +0000 Subject: [PATCH 212/409] changelogs: Mark version 20.4.0 and 20.8.0 do not use Problem We have two versions in the past are released with corrupt jars. Solution With patch released fixed the issue, we want to mark the problematic releases to avoid usage. JIRA Issues: CSL-10338 Differential Revision: https://phabricator.twitter.biz/D579740 --- CHANGELOG.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6ed485a1..79386d2d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -41,8 +41,8 @@ Breaking API Changes * Check SecurityManager permissions in the `ContentHandler` to ensure that contention snapshotting is allowed. ``PHAB_ID=D531873`` -20.8.0 ------- +20.8.0 (DO NOT USE) +------------------- No Changes @@ -82,8 +82,8 @@ Runtime Behavior Changes No Changes -20.4.0 ------- +20.4.0 (DO NOT USE) +------------------- No Changes From 233976d4defef7c093397cb188640f5766797ab8 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Thu, 10 Dec 2020 18:25:11 +0000 Subject: [PATCH 213/409] Twitter-oss: Prepare OSS libraries for release 20.12.0 Problem We want to release the next versions of our Twitter OSS libraries 20.12.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-10389 Differential Revision: https://phabricator.twitter.biz/D589851 --- CHANGELOG.rst | 3 +++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 79386d2d..aa34be62 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +20.12.0 +------- + * Escape user-provided string from the returned text. This removes a potential vector for an XSS attack. ``PHAB_ID=D574844`` diff --git a/README.md b/README.md index 3fac126b..37015aec 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.com/twitter/twitter-server.svg?branch=develop)](https://travis-ci.com/twitter/twitter-server) +[![Build status](https://travis-ci.com/twitter/twitter-server.svg?branch=release)](https://travis-ci.com/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index bccf8551..e59dd95c 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.11.0-SNAPSHOT" +val releaseVersion = "20.12.0" val jacksonVersion = "2.11.2" val jacksonLibs = Seq( From 1660132bf380bd094bc7748325925f9f1ae804d9 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Thu, 10 Dec 2020 21:07:12 +0000 Subject: [PATCH 214/409] Twitter-oss: update OSS libraries post-release to 21.1.0 Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 21.1.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. JIRA Issues: CSL-10389 Differential Revision: https://phabricator.twitter.biz/D591241 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 37015aec..3fac126b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.com/twitter/twitter-server.svg?branch=release)](https://travis-ci.com/twitter/twitter-server) +[![Build status](https://travis-ci.com/twitter/twitter-server.svg?branch=develop)](https://travis-ci.com/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index e59dd95c..d62688b2 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.12.0" +val releaseVersion = "21.1.0-SNAPSHOT" val jacksonVersion = "2.11.2" val jacksonLibs = Seq( From 38a437b832f437bca18983ad9527190054618eae Mon Sep 17 00:00:00 2001 From: Matt Dannenberg Date: Fri, 11 Dec 2020 20:16:14 +0000 Subject: [PATCH 215/409] twitter-server: add scope separator character to metric_metadata response Problem metric_metadata.json does not return the scope separating character and consequently consumers of the endpoint would have to infer the character from full metric names. Solution Expose the scope separating character as a top level "separator_char" field in metrics_metadata.json. Result Consumers of the metric_metadata.json endpoint can now determine the scope separating character via the top level field. JIRA Issues: CSL-10255 Differential Revision: https://phabricator.twitter.biz/D591106 --- CHANGELOG.rst | 3 +++ .../twitter/server/handler/MetricMetadataQueryHandler.scala | 4 +++- .../server/handler/MetricMetadataQueryHandlerTest.scala | 6 ++++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index aa34be62..fd5eabb2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +* Add separator_char field to the top level of the Metrics Metadata endpoint and bump the + endpoints version number to 2.1. ``PHAB_ID=D591106`` + 20.12.0 ------- diff --git a/server/src/main/scala/com/twitter/server/handler/MetricMetadataQueryHandler.scala b/server/src/main/scala/com/twitter/server/handler/MetricMetadataQueryHandler.scala index 76907717..713774cb 100644 --- a/server/src/main/scala/com/twitter/server/handler/MetricMetadataQueryHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/MetricMetadataQueryHandler.scala @@ -2,6 +2,7 @@ package com.twitter.server.handler import com.twitter.finagle.Service import com.twitter.finagle.http.{MediaType, Request, Response, Uri} +import com.twitter.finagle.stats.metadataScopeSeparator import com.twitter.io.Buf import com.twitter.server.util.HttpUtils.newResponse import com.twitter.server.util.{JsonConverter, MetricSchemaSource} @@ -95,8 +96,9 @@ class MetricMetadataQueryHandler(source: MetricSchemaSource = new MetricSchemaSo content = Buf.Utf8( JsonConverter.writeToString( Map( - "@version" -> 2.0, + "@version" -> 2.1, "counters_latched" -> latched, + "separator_char" -> metadataScopeSeparator(), "metrics" -> metrics ))) ) diff --git a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala index 722fe6a3..362def8e 100644 --- a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala @@ -307,8 +307,9 @@ class MetricMetadataQueryHandlerTest extends FunSuite { val responseStart = """ | { - | "@version" : 2.0, + | "@version" : 2.1, | "counters_latched" : true, + | "separator_char" : "/", """.stripMargin test(testName + " when using latched counters") { assertJsonResponse( @@ -320,8 +321,9 @@ class MetricMetadataQueryHandlerTest extends FunSuite { val responseStart = """ | { - | "@version" : 2.0, + | "@version" : 2.1, | "counters_latched" : false, + | "separator_char" : "/", """.stripMargin test(testName + " when using unlatched counters") { assertJsonResponse( From 9cacdfbafefe4f9531483e93927bdfd5382167f8 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Wed, 13 Jan 2021 21:45:46 +0000 Subject: [PATCH 216/409] Twitter-oss: Prepare OSS libraries for release 21.1.0 Problem We want to release the next versions of our Twitter OSS libraries 21.1.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D602794 --- CHANGELOG.rst | 3 +++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index fd5eabb2..3468adfe 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +21.1.0 +------ + * Add separator_char field to the top level of the Metrics Metadata endpoint and bump the endpoints version number to 2.1. ``PHAB_ID=D591106`` diff --git a/README.md b/README.md index 3fac126b..37015aec 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.com/twitter/twitter-server.svg?branch=develop)](https://travis-ci.com/twitter/twitter-server) +[![Build status](https://travis-ci.com/twitter/twitter-server.svg?branch=release)](https://travis-ci.com/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index d62688b2..8724147a 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.1.0-SNAPSHOT" +val releaseVersion = "21.1.0" val jacksonVersion = "2.11.2" val jacksonLibs = Seq( From 9e7a2fe5a1bc401fd93b5fd4b4c4a2b7b5a061b0 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Thu, 14 Jan 2021 23:34:19 +0000 Subject: [PATCH 217/409] Twitter-oss: update OSS libraries post-release to 20.2.0 Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 20.2.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D603517 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 37015aec..3fac126b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.com/twitter/twitter-server.svg?branch=release)](https://travis-ci.com/twitter/twitter-server) +[![Build status](https://travis-ci.com/twitter/twitter-server.svg?branch=develop)](https://travis-ci.com/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 8724147a..9cbd487e 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.1.0" +val releaseVersion = "21.2.0-SNAPSHOT" val jacksonVersion = "2.11.2" val jacksonLibs = Seq( From ff232351a9b4e9795c4c54aff5354249aca5420d Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Thu, 21 Jan 2021 01:00:29 +0000 Subject: [PATCH 218/409] twitter-server: add null handler for writing metric metadata relative_name Problem In some cases, metrics are scoped with `null`. MetricMetadataQueryhandler writes them as literal `null`s which causes issues for consumers. Solution Write null as string "null". JIRA Issues: CSL-10572 Differential Revision: https://phabricator.twitter.biz/D604911 --- .../server/util/MetricSchemaJsonModule.scala | 11 ++++++-- .../MetricMetadataQueryHandlerTest.scala | 27 +++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/util/MetricSchemaJsonModule.scala b/server/src/main/scala/com/twitter/server/util/MetricSchemaJsonModule.scala index 6e38f202..000e35ff 100644 --- a/server/src/main/scala/com/twitter/server/util/MetricSchemaJsonModule.scala +++ b/server/src/main/scala/com/twitter/server/util/MetricSchemaJsonModule.scala @@ -37,9 +37,11 @@ object SchemaSerializer extends StdSerializer[MetricSchema](classOf[MetricSchema metricSchema.metricBuilder.name.mkString(metadataScopeSeparator())) jsonGenerator.writeArrayFieldStart("relative_name") if (metricSchema.metricBuilder.relativeName != Seq.empty) { - metricSchema.metricBuilder.relativeName.foreach(segment => jsonGenerator.writeString(segment)) + metricSchema.metricBuilder.relativeName.foreach(segment => + jsonGenerator.writeString(convertNullToString(segment))) } else { - metricSchema.metricBuilder.name.foreach(segment => jsonGenerator.writeString(segment)) + metricSchema.metricBuilder.name.foreach(segment => + jsonGenerator.writeString(convertNullToString(segment))) } jsonGenerator.writeEndArray() val dataType = metricSchema match { @@ -68,6 +70,11 @@ object SchemaSerializer extends StdSerializer[MetricSchema](classOf[MetricSchema } jsonGenerator.writeEndObject() } + + private def convertNullToString(segment: String): String = { + if (segment == null) "null" + else segment + } } object MetricSchemaJsonModule extends SimpleModule { diff --git a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala index 362def8e..e43729b1 100644 --- a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala @@ -40,6 +40,19 @@ class MetricMetadataQueryHandlerTest extends FunSuite { name = Seq("my", "only", "histo"), percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), statsReceiver = null + )), + "my/bad/null/counter" -> CounterSchema( + new MetricBuilder( + keyIndicator = true, + description = "A counter scoped by null get deserialized correctly", + units = Requests, + role = Server, + verbosity = Verbosity.Default, + sourceClass = Some("finagle.stats.bad"), + name = Seq("my", "bad", null, "counter"), + processPath = Some("dc/role/zone/service"), + percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), + statsReceiver = null )) ) @@ -136,6 +149,20 @@ class MetricMetadataQueryHandlerTest extends FunSuite { | 0.999, | 0.9999 | ] + | }, + | { + | "name" : "my/bad/null/counter", + | "relative_name" : ["my","bad", "null", "counter"], + | "kind" : "counter", + | "source" : { + | "class": "finagle.stats.bad", + | "category": "Server", + | "process_path": "dc/role/zone/service" + | }, + | "description" : "A counter scoped by null get deserialized correctly", + | "unit" : "Requests", + | "verbosity": "Verbosity(default)", + | "key_indicator" : true | } | ] | } From d4b704321ed14f5f41f76d36575e57d242fd70fd Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Sat, 23 Jan 2021 02:47:31 +0000 Subject: [PATCH 219/409] oss: Update README copyright year to 2021 Problem We've crossed into a new year and our copyright ranges are now out of date. Solution Update them to 2021. JIRA Issues: CSL-10577 Differential Revision: https://phabricator.twitter.biz/D606603 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3fac126b..b82696e7 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,6 @@ for more details about how to contribute. ## License -Copyright 2013-2020 Twitter, Inc. +Copyright 2013-2021 Twitter, Inc. Licensed under the Apache License, Version 2.0: https://www.apache.org/licenses/LICENSE-2.0 From 91220e59b52b269bd1809ea6fb3c2996267016da Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Mon, 1 Feb 2021 19:45:09 +0000 Subject: [PATCH 220/409] csl: Remove ryano from Owners/Project Files Differential Revision: https://phabricator.twitter.biz/D610849 --- PROJECT.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/PROJECT.yml b/PROJECT.yml index 8ee3c888..5cd33c60 100644 --- a/PROJECT.yml +++ b/PROJECT.yml @@ -7,6 +7,5 @@ owners: - koliver - mnakamura - roanta - - ryano watchers: - csl-team:ldap From eb836e4afb00ea29ac8dd2256045514e86870830 Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Thu, 4 Feb 2021 22:14:09 +0000 Subject: [PATCH 221/409] oss: Remove Date Ranges from Copyright in READMEs Problem/Solution: The copyright lines in the project README files, do not need to be date ranges but can simply be the original (first) year the code was copyrighted. JIRA Issues: CSL-10588 Differential Revision: https://phabricator.twitter.biz/D613522 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b82696e7..fbf1e471 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,6 @@ for more details about how to contribute. ## License -Copyright 2013-2021 Twitter, Inc. +Copyright 2013 Twitter, Inc. Licensed under the Apache License, Version 2.0: https://www.apache.org/licenses/LICENSE-2.0 From 34613c03d8c579c4dd3314b23118f2032166cb41 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Wed, 10 Feb 2021 22:33:04 +0000 Subject: [PATCH 222/409] Twitter-oss: Prepare OSS libraries for release 21.2.0 Problem We want to release the next versions of our Twitter OSS libraries 21.2.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D615824 --- CHANGELOG.rst | 5 +++++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3468adfe..6931e191 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +21.2.0 +------ + +No Changes + 21.1.0 ------ diff --git a/README.md b/README.md index fbf1e471..a831c119 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.com/twitter/twitter-server.svg?branch=develop)](https://travis-ci.com/twitter/twitter-server) +[![Build status](https://travis-ci.com/twitter/twitter-server.svg?branch=release)](https://travis-ci.com/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 9cbd487e..aa0abccd 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.2.0-SNAPSHOT" +val releaseVersion = "21.2.0" val jacksonVersion = "2.11.2" val jacksonLibs = Seq( From adcda839c2f05b2ba12b87773b187475da6a30ee Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Wed, 10 Feb 2021 23:48:22 +0000 Subject: [PATCH 223/409] Twitter-oss: update OSS libraries post-release to 21.3.0 Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 21.3.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D617097 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a831c119..fbf1e471 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.com/twitter/twitter-server.svg?branch=release)](https://travis-ci.com/twitter/twitter-server) +[![Build status](https://travis-ci.com/twitter/twitter-server.svg?branch=develop)](https://travis-ci.com/twitter/twitter-server) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index aa0abccd..de852e66 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.2.0" +val releaseVersion = "21.3.0-SNAPSHOT" val jacksonVersion = "2.11.2" val jacksonLibs = Seq( From a674f9239e8d22f9ed210963af247fdebcc3e1bb Mon Sep 17 00:00:00 2001 From: Hamdi Allam Date: Wed, 17 Feb 2021 20:09:17 +0000 Subject: [PATCH 224/409] [finagle/util/twitter-server] add hallam as owner to projects Problem: hallam should own some projects. Solution / Result: Add hallam. Add banderson to finagle-thriftmux Differential Revision: https://phabricator.twitter.biz/D619360 --- PROJECT.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/PROJECT.yml b/PROJECT.yml index 5cd33c60..1643d89d 100644 --- a/PROJECT.yml +++ b/PROJECT.yml @@ -2,6 +2,7 @@ owners: - banderson + - hallam - ccoco - ibennett - koliver From 4d15ed7d5bbfcf839c22314c26640be9440742e6 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Sat, 20 Feb 2021 01:08:08 +0000 Subject: [PATCH 225/409] twitter-server: Add GitHub Action continuous integration Problem/Solution Add continuous integration with a GitHub Action. Differential Revision: https://phabricator.twitter.biz/D621378 --- .github/workflows/continuous-integration.yml | 68 ++++++++++++++++++++ README.md | 2 +- 2 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/continuous-integration.yml diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml new file mode 100644 index 00000000..108e0605 --- /dev/null +++ b/.github/workflows/continuous-integration.yml @@ -0,0 +1,68 @@ +name: continuous integration + +env: + JAVA_OPTS: "-Dsbt.log.noformat=true" + TRAVIS: "true" # pretend we're TravisCI + +defaults: + run: + shell: bash +on: + push: + branches: + - develop + - release + pull_request: + +jobs: + build: + strategy: + fail-fast: false + matrix: + scala: [2.11.12, 2.12.8, 2.13.1] + java: ['1.8', '1.11'] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2.3.4 + - uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + - name: echo java version + run: java -Xmx32m -version + - name: echo javac version + run: javac -J-Xmx32m -version + - name: cache build dependencies + uses: actions/cache@v2 + env: + cache-name: cache-build-deps + with: + path: | + ~/.dodo + ~/.ivy2/cache + ~/.ivy2/local/com.twitter + ~/.sbt + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ matrix.scala }}-${{ matrix.java }} + - name: update cache + run: | + if [ -f ~/.ivy2/cache ]; then + find ~/.ivy2/cache -name "ivydata-*.properties" -delete + fi + if [ -f ~/.sbt ]; then + find ~/.sbt -name "*.lock" -delete + fi + - name: build dependencies + uses: twitter/dodo-build@v3 + with: + branch: develop + project: twitter-server + no-test: true + publish-m2: true + verbose: true + - name: test + run: ${{ format('./sbt ++{0} clean coverage test coverageReport', matrix.scala) }} + - name: coverage + if: ${{ success() }} + run: ${{ format('./sbt ++{0} coverageAggregate', matrix.scala) }} + - name: upload coverage + if: ${{ success() }} + run: bash <(curl -s https://codecov.io/bash) diff --git a/README.md b/README.md index fbf1e471..90042271 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build status](https://travis-ci.com/twitter/twitter-server.svg?branch=develop)](https://travis-ci.com/twitter/twitter-server) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) From c2386e2c329f6c9a13e041b0d8799f964270e65e Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Thu, 4 Mar 2021 22:09:45 +0000 Subject: [PATCH 226/409] Twitter-oss: Remove TravisCI configuration for projects Problem/Solution We have updated to using GitHub workflows for continuous integration. Let's remove the TravisCI configuration. JIRA Issues: CSL-8182 Differential Revision: https://phabricator.twitter.biz/D628740 --- .travis.yml | 55 ----------------------------------------------------- 1 file changed, 55 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 6f8b7c00..00000000 --- a/.travis.yml +++ /dev/null @@ -1,55 +0,0 @@ -dist: trusty -sudo: required - -language: scala - -git: - depth: 3 - quiet: true - -env: - - JAVA_OPTS="-Dsbt.log.noformat=true" - -# These directories are cached to S3 at the end of the build -cache: - directories: - - $HOME/.ivy2/cache - - $HOME/.ivy2/local/com.twitter - - $HOME/.sbt - - $HOME/.dodo - -before_cache: - # Cleanup the cached directories to avoid unnecessary cache updates - - find $HOME/.ivy2/cache -name "ivydata-*.properties" -delete - - find $HOME/.sbt -name "*.lock" -delete - -scala: - - 2.11.12 - - 2.12.8 - - 2.13.1 - -jdk: - - openjdk8 - - openjdk11 - -matrix: - # Does not allow for assigning a build result if the non-'allow_failures' targets are complete - # because of https://travis-ci.community/t/receiving-too-many-emails/4888 - fast_finish: false - -notifications: - slack: - secure: FfzYHh8IvtBaYswh9mIQU9yQGr9YpkRaFR77RaYdK5wVhw3UlCBtRTBdFpUn0djgg3Pl3FxEj5OyEKYQ803xxF7UQRxx298g4r5OhOzjQ4UH1lhnAg1WpIRPBL4DuWrreZFtnqvsygrdtvATZ1aLzaYtkaCV49x/t2Kr7weNfpQ= - on_failure: change - on_success: change - -before_script: - - unset SBT_OPTS # default $SBT_OPTS is irrelevant to sbt launcher - - curl -s https://raw.githubusercontent.com/twitter/dodo/develop/bin/build | bash -s -- --no-test --scala-version $TRAVIS_SCALA_VERSION twitter-server - -script: - - travis_retry ./sbt ++$TRAVIS_SCALA_VERSION clean coverage test coverageReport - -after_success: - - ./sbt ++$TRAVIS_SCALA_VERSION coverageAggregate - - bash <(curl -s https://codecov.io/bash) From 5e15ab5b78204af99da96c9e44ebb9fa4e437a4f Mon Sep 17 00:00:00 2001 From: Matt Dannenberg Date: Fri, 5 Mar 2021 21:11:34 +0000 Subject: [PATCH 227/409] twitter-server: add histogram bucket metric names to metric_metadata.json Problem Consumers of the metric_metadata.json endpoint do not have a good way of discovering the metrics which make up a histogram entry in metric_metadata.json. Solution Replace the array type bucket field of metric_metadata.json (which simply listed the percentiles) with an Object field mapping from names (min, max, etc) and percentiles to the suffix for the metric name representing that portion of the histogram. Result The metric_metadata.json output now has a map from percentiles et al to the suffix for the metric name for that percentile (or max, min, etc). JIRA Issues: CSL-10261 Differential Revision: https://phabricator.twitter.biz/D627870 --- CHANGELOG.rst | 4 +- build.sbt | 1 + .../handler/MetricMetadataQueryHandler.scala | 24 ++++--- .../server/util/MetricSchemaJsonModule.scala | 26 +++++-- .../MetricMetadataQueryHandlerTest.scala | 67 ++++++++++++------- .../util/MetricSchemaJsonModuleTest.scala | 15 ++++- 6 files changed, 94 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6931e191..3ebdc156 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,7 +10,9 @@ Unreleased 21.2.0 ------ -No Changes +* Change Metrics Metadata histogram buckets field to be an object mapping from names (min, max, etc) + and percentiles to the suffix for the metric name representing that portion of the histogram. + Bump the endpoint's version number to 3.0. ``PHAB_ID=D627870`` 21.1.0 ------ diff --git a/build.sbt b/build.sbt index de852e66..4ca1be9a 100644 --- a/build.sbt +++ b/build.sbt @@ -177,6 +177,7 @@ lazy val twitterServer = (project in file("server")) "org.slf4j" % "slf4j-api" % slf4jVersion, finagle("core"), finagle("http"), + finagle("stats-core"), finagle("toggle"), finagle("tunable"), util("app"), diff --git a/server/src/main/scala/com/twitter/server/handler/MetricMetadataQueryHandler.scala b/server/src/main/scala/com/twitter/server/handler/MetricMetadataQueryHandler.scala index 713774cb..717c7878 100644 --- a/server/src/main/scala/com/twitter/server/handler/MetricMetadataQueryHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/MetricMetadataQueryHandler.scala @@ -21,7 +21,7 @@ import com.twitter.util.Future * * Response: * { - * "@version" : 2.0, + * "@version" : 3.0, * "counters_latched" : false, * "metrics" : [ * { @@ -65,13 +65,19 @@ import com.twitter.util.Future * "unit" : "Unspecified", * "verbosity": "Verbosity(default)", * "key_indicator" : false, - * "buckets" : [ - * 0.5, - * 0.9, - * 0.99, - * 0.999, - * 0.9999 - * ] + * "buckets" : { + * "count" : ".count", + * "sum" : ".sum", + * "average" : ".avg", + * "minimum" : ".min", + * "maximum" : ".max", + * "0.5" : ".p50", + * "0.9" : ".p90", + * "0.95" : ".p95", + * "0.99" : ".p99", + * "0.999" : ".p9990", + * "0.9999" : ".p9999" + * } * } * ]} */ @@ -96,7 +102,7 @@ class MetricMetadataQueryHandler(source: MetricSchemaSource = new MetricSchemaSo content = Buf.Utf8( JsonConverter.writeToString( Map( - "@version" -> 2.1, + "@version" -> 3.0, "counters_latched" -> latched, "separator_char" -> metadataScopeSeparator(), "metrics" -> metrics diff --git a/server/src/main/scala/com/twitter/server/util/MetricSchemaJsonModule.scala b/server/src/main/scala/com/twitter/server/util/MetricSchemaJsonModule.scala index 000e35ff..84f5f6f9 100644 --- a/server/src/main/scala/com/twitter/server/util/MetricSchemaJsonModule.scala +++ b/server/src/main/scala/com/twitter/server/util/MetricSchemaJsonModule.scala @@ -9,11 +9,14 @@ import com.twitter.finagle.stats.{ GaugeSchema, HistogramSchema, MetricSchema, + StatsFormatter, metadataScopeSeparator } object SchemaSerializer extends StdSerializer[MetricSchema](classOf[MetricSchema]) { + private[this] val statsFormatter = StatsFormatter.default + /** * This custom serializer is used to convert MetricSchemas to JSON for the metric_metadata * endpoint. @@ -31,10 +34,9 @@ object SchemaSerializer extends StdSerializer[MetricSchema](classOf[MetricSchema jsonGenerator: JsonGenerator, serializerProvider: SerializerProvider ): Unit = { + val formattedName = metricSchema.metricBuilder.name.mkString(metadataScopeSeparator()) jsonGenerator.writeStartObject() - jsonGenerator.writeStringField( - "name", - metricSchema.metricBuilder.name.mkString(metadataScopeSeparator())) + jsonGenerator.writeStringField("name", formattedName) jsonGenerator.writeArrayFieldStart("relative_name") if (metricSchema.metricBuilder.relativeName != Seq.empty) { metricSchema.metricBuilder.relativeName.foreach(segment => @@ -63,10 +65,22 @@ object SchemaSerializer extends StdSerializer[MetricSchema](classOf[MetricSchema jsonGenerator.writeStringField("unit", metricSchema.metricBuilder.units.toString) jsonGenerator.writeStringField("verbosity", metricSchema.metricBuilder.verbosity.toString) jsonGenerator.writeBooleanField("key_indicator", metricSchema.metricBuilder.keyIndicator) + if (metricSchema.isInstanceOf[HistogramSchema]) { - jsonGenerator.writeArrayFieldStart("buckets") - metricSchema.metricBuilder.percentiles.foreach(bucket => jsonGenerator.writeNumber(bucket)) - jsonGenerator.writeEndArray() + jsonGenerator.writeObjectFieldStart("buckets") + jsonGenerator.writeStringField("count", statsFormatter.separator + "count") + jsonGenerator.writeStringField("sum", statsFormatter.separator + "sum") + jsonGenerator.writeStringField( + "average", + statsFormatter.separator + statsFormatter.labelAverage) + jsonGenerator.writeStringField("minimum", statsFormatter.separator + statsFormatter.labelMin) + jsonGenerator.writeStringField("maximum", statsFormatter.separator + statsFormatter.labelMax) + metricSchema.metricBuilder.percentiles.foreach(bucket => + jsonGenerator.writeStringField( + bucket.toString, + statsFormatter.separator + statsFormatter.labelPercentile(bucket))) + + jsonGenerator.writeEndObject() } jsonGenerator.writeEndObject() } diff --git a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala index e43729b1..7cc34ca8 100644 --- a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala @@ -141,14 +141,19 @@ class MetricMetadataQueryHandlerTest extends FunSuite { | "unit" : "Unspecified", | "verbosity": "Verbosity(default)", | "key_indicator" : false, - | "buckets" : [ - | 0.5, - | 0.9, - | 0.95, - | 0.99, - | 0.999, - | 0.9999 - | ] + | "buckets" : { + | "count" : ".count", + | "sum" : ".sum", + | "average" : ".avg", + | "minimum" : ".min", + | "maximum" : ".max", + | "0.5" : ".p50", + | "0.9" : ".p90", + | "0.95" : ".p95", + | "0.99" : ".p99", + | "0.999" : ".p9990", + | "0.9999" : ".p9999" + | } | }, | { | "name" : "my/bad/null/counter", @@ -239,14 +244,19 @@ class MetricMetadataQueryHandlerTest extends FunSuite { | "unit" : "Unspecified", | "verbosity": "Verbosity(default)", | "key_indicator" : false, - | "buckets" : [ - | 0.5, - | 0.9, - | 0.95, - | 0.99, - | 0.999, - | 0.9999 - | ] + | "buckets" : { + | "count" : ".count", + | "sum" : ".sum", + | "average" : ".avg", + | "minimum" : ".min", + | "maximum" : ".max", + | "0.5" : ".p50", + | "0.9" : ".p90", + | "0.95" : ".p95", + | "0.99" : ".p99", + | "0.999" : ".p9990", + | "0.9999" : ".p9999" + | } | } | ] | } """.stripMargin @@ -281,14 +291,19 @@ class MetricMetadataQueryHandlerTest extends FunSuite { | "unit" : "Unspecified", | "verbosity": "Verbosity(default)", | "key_indicator" : false, - | "buckets" : [ - | 0.5, - | 0.9, - | 0.95, - | 0.99, - | 0.999, - | 0.9999 - | ] + | "buckets" : { + | "count" : ".count", + | "sum" : ".sum", + | "average" : ".avg", + | "minimum" : ".min", + | "maximum" : ".max", + | "0.5" : ".p50", + | "0.9" : ".p90", + | "0.95" : ".p95", + | "0.99" : ".p99", + | "0.999" : ".p9990", + | "0.9999" : ".p9999" + | } | } | ] | } @@ -334,7 +349,7 @@ class MetricMetadataQueryHandlerTest extends FunSuite { val responseStart = """ | { - | "@version" : 2.1, + | "@version" : 3.0, | "counters_latched" : true, | "separator_char" : "/", """.stripMargin @@ -348,7 +363,7 @@ class MetricMetadataQueryHandlerTest extends FunSuite { val responseStart = """ | { - | "@version" : 2.1, + | "@version" : 3.0, | "counters_latched" : false, | "separator_char" : "/", """.stripMargin diff --git a/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala b/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala index 25bd0e32..2e2bc192 100644 --- a/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala +++ b/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala @@ -92,7 +92,20 @@ class MetricSchemaJsonModuleTest extends FunSuite { "category" -> "NoRoleSpecified", "process_path" -> "Unspecified")) assert(jsonMap.get("kind").get == "histogram") - assert(jsonMap.get("buckets").get == IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999)) + assert( + jsonMap.get("buckets").get == Map( + "count" -> ".count", + "0.5" -> ".p50", + "0.99" -> ".p99", + "0.999" -> ".p9990", + "0.95" -> ".p95", + "0.9" -> ".p90", + "0.9999" -> ".p9999", + "maximum" -> ".max", + "average" -> ".avg", + "minimum" -> ".min", + "sum" -> ".sum" + )) } } From e2676f0f219ab4de1bca3827316d0444c2e1acf0 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Mon, 8 Mar 2021 20:10:08 +0000 Subject: [PATCH 228/409] {finagle, util, twitter-server}: Metric Expression prototype Problem We want to ship our prototype of Metric Expression to start the development, this prototype confirms that end-to-end the expression instrument and exports work as we expected. Solution The current prototype code is sitting in experiment packages, the exported expression json document is under-development. JIRA Issues: CSL-10616 Differential Revision: https://phabricator.twitter.biz/D615453 --- .../main/scala/com/twitter/server/Admin.scala | 17 +-- .../handler/MetricExpressionHandler.scala | 24 ++++ .../server/util/MetricSchemaJsonModule.scala | 91 +------------ .../server/util/MetricSchemaSerializer.scala | 91 +++++++++++++ .../server/util/MetricSchemaSource.scala | 5 + .../util/exp/ExpressionSchemaSerializer.scala | 65 +++++++++ .../MetricMetadataQueryHandlerTest.scala | 5 +- .../exp/MetricExpressionHandlerTest.scala | 123 ++++++++++++++++++ .../server/util/MetricSchemaSourceTest.scala | 11 +- 9 files changed, 328 insertions(+), 104 deletions(-) create mode 100644 server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala create mode 100644 server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala create mode 100644 server/src/main/scala/com/twitter/server/util/exp/ExpressionSchemaSerializer.scala create mode 100644 server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala diff --git a/server/src/main/scala/com/twitter/server/Admin.scala b/server/src/main/scala/com/twitter/server/Admin.scala index 39b3d328..366df7b6 100644 --- a/server/src/main/scala/com/twitter/server/Admin.scala +++ b/server/src/main/scala/com/twitter/server/Admin.scala @@ -168,6 +168,13 @@ object Admin { group = Some(Grouping.Metrics), includeInIndex = true ), + Route( + path = "/admin/metric/expressions.json", + handler = new MetricExpressionHandler(), + alias = "Metric Expressions", + group = Some(Grouping.Metrics), + includeInIndex = true + ), Route( path = Path.Clients, handler = new ClientRegistryHandler(Path.Clients), @@ -276,16 +283,6 @@ object Admin { } case None => Nil } - } ++ { - Seq( - Route( - path = "/admin/exp/metric_metadata", - handler = new MetricTypeQueryHandler(details = aggregate), - alias = "Metric Metadata", - group = Some(Grouping.Metrics), - includeInIndex = false - ) - ) } } diff --git a/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala b/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala new file mode 100644 index 00000000..a61ec7c5 --- /dev/null +++ b/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala @@ -0,0 +1,24 @@ +package com.twitter.server.handler + +import com.twitter.finagle.Service +import com.twitter.finagle.http.{MediaType, Request, Response} +import com.twitter.io.Buf +import com.twitter.server.util.HttpUtils.newResponse +import com.twitter.server.util.{JsonConverter, MetricSchemaSource} +import com.twitter.util.Future + +class MetricExpressionHandler(source: MetricSchemaSource = new MetricSchemaSource) + extends Service[Request, Response] { + + def apply(request: Request): Future[Response] = { + val expressions = source.expressionList + newResponse( + contentType = MediaType.JsonUtf8, + content = Buf.Utf8( + JsonConverter.writeToString( + Map("@version" -> 0.1, "expressions" -> expressions) + ) + ) + ) + } +} diff --git a/server/src/main/scala/com/twitter/server/util/MetricSchemaJsonModule.scala b/server/src/main/scala/com/twitter/server/util/MetricSchemaJsonModule.scala index 84f5f6f9..fb474d25 100644 --- a/server/src/main/scala/com/twitter/server/util/MetricSchemaJsonModule.scala +++ b/server/src/main/scala/com/twitter/server/util/MetricSchemaJsonModule.scala @@ -1,96 +1,9 @@ package com.twitter.server.util -import com.fasterxml.jackson.core.JsonGenerator -import com.fasterxml.jackson.databind.SerializerProvider import com.fasterxml.jackson.databind.module.SimpleModule -import com.fasterxml.jackson.databind.ser.std.StdSerializer -import com.twitter.finagle.stats.{ - CounterSchema, - GaugeSchema, - HistogramSchema, - MetricSchema, - StatsFormatter, - metadataScopeSeparator -} - -object SchemaSerializer extends StdSerializer[MetricSchema](classOf[MetricSchema]) { - - private[this] val statsFormatter = StatsFormatter.default - - /** - * This custom serializer is used to convert MetricSchemas to JSON for the metric_metadata - * endpoint. - */ -// The impetus for a customer serializer over case class serde is: -// 1) The nested nature of MetricBuilder with MetricSchema means we do not have the ability to -// make decisions about which MetricBuilder fields to include based on MetricSchema type -// (ie, buckets should only be present for HistogramSchema). -// 2) Reshaping the MetricBuilder to be the top level JSON object removes the ability to inject -// the "kind" field based on the MetricSchema type. -// 3) The MetricBuilder class would need to be reworked to have a Source case class -// nested in it which would contain a few of the currently MetricBuilder level values. - def serialize( - metricSchema: MetricSchema, - jsonGenerator: JsonGenerator, - serializerProvider: SerializerProvider - ): Unit = { - val formattedName = metricSchema.metricBuilder.name.mkString(metadataScopeSeparator()) - jsonGenerator.writeStartObject() - jsonGenerator.writeStringField("name", formattedName) - jsonGenerator.writeArrayFieldStart("relative_name") - if (metricSchema.metricBuilder.relativeName != Seq.empty) { - metricSchema.metricBuilder.relativeName.foreach(segment => - jsonGenerator.writeString(convertNullToString(segment))) - } else { - metricSchema.metricBuilder.name.foreach(segment => - jsonGenerator.writeString(convertNullToString(segment))) - } - jsonGenerator.writeEndArray() - val dataType = metricSchema match { - case _: CounterSchema => "counter" - case _: GaugeSchema => "gauge" - case _: HistogramSchema => "histogram" - } - jsonGenerator.writeStringField("kind", dataType) - jsonGenerator.writeObjectFieldStart("source") - jsonGenerator.writeStringField( - "class", - metricSchema.metricBuilder.sourceClass.getOrElse("Unspecified")) - jsonGenerator.writeStringField("category", metricSchema.metricBuilder.role.toString) - jsonGenerator.writeStringField( - "process_path", - metricSchema.metricBuilder.processPath.getOrElse("Unspecified")) - jsonGenerator.writeEndObject() - jsonGenerator.writeStringField("description", metricSchema.metricBuilder.description) - jsonGenerator.writeStringField("unit", metricSchema.metricBuilder.units.toString) - jsonGenerator.writeStringField("verbosity", metricSchema.metricBuilder.verbosity.toString) - jsonGenerator.writeBooleanField("key_indicator", metricSchema.metricBuilder.keyIndicator) - - if (metricSchema.isInstanceOf[HistogramSchema]) { - jsonGenerator.writeObjectFieldStart("buckets") - jsonGenerator.writeStringField("count", statsFormatter.separator + "count") - jsonGenerator.writeStringField("sum", statsFormatter.separator + "sum") - jsonGenerator.writeStringField( - "average", - statsFormatter.separator + statsFormatter.labelAverage) - jsonGenerator.writeStringField("minimum", statsFormatter.separator + statsFormatter.labelMin) - jsonGenerator.writeStringField("maximum", statsFormatter.separator + statsFormatter.labelMax) - metricSchema.metricBuilder.percentiles.foreach(bucket => - jsonGenerator.writeStringField( - bucket.toString, - statsFormatter.separator + statsFormatter.labelPercentile(bucket))) - - jsonGenerator.writeEndObject() - } - jsonGenerator.writeEndObject() - } - - private def convertNullToString(segment: String): String = { - if (segment == null) "null" - else segment - } -} +import com.twitter.server.util.exp.ExpressionSchemaSerializer object MetricSchemaJsonModule extends SimpleModule { addSerializer(SchemaSerializer) + addSerializer(ExpressionSchemaSerializer) } diff --git a/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala b/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala new file mode 100644 index 00000000..d2892534 --- /dev/null +++ b/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala @@ -0,0 +1,91 @@ +package com.twitter.server.util + +import com.fasterxml.jackson.core.JsonGenerator +import com.fasterxml.jackson.databind.SerializerProvider +import com.fasterxml.jackson.databind.ser.std.StdSerializer +import com.twitter.finagle.stats.{ + CounterSchema, + GaugeSchema, + HistogramSchema, + MetricSchema, + StatsFormatter, + metadataScopeSeparator +} + +object SchemaSerializer extends StdSerializer[MetricSchema](classOf[MetricSchema]) { + + private[this] val statsFormatter = StatsFormatter.default + + /** + * This custom serializer is used to convert MetricSchemas to JSON for the metric_metadata + * endpoint. + */ + // The impetus for a customer serializer over case class serde is: + // 1) The nested nature of MetricBuilder with MetricSchema means we do not have the ability to + // make decisions about which MetricBuilder fields to include based on MetricSchema type + // (ie, buckets should only be present for HistogramSchema). + // 2) Reshaping the MetricBuilder to be the top level JSON object removes the ability to inject + // the "kind" field based on the MetricSchema type. + // 3) The MetricBuilder class would need to be reworked to have a Source case class + // nested in it which would contain a few of the currently MetricBuilder level values. + def serialize( + metricSchema: MetricSchema, + jsonGenerator: JsonGenerator, + serializerProvider: SerializerProvider + ): Unit = { + val formattedName = metricSchema.metricBuilder.name.mkString(metadataScopeSeparator()) + jsonGenerator.writeStartObject() + jsonGenerator.writeStringField("name", formattedName) + jsonGenerator.writeArrayFieldStart("relative_name") + if (metricSchema.metricBuilder.relativeName != Seq.empty) { + metricSchema.metricBuilder.relativeName.foreach(segment => + jsonGenerator.writeString(convertNullToString(segment))) + } else { + metricSchema.metricBuilder.name.foreach(segment => + jsonGenerator.writeString(convertNullToString(segment))) + } + jsonGenerator.writeEndArray() + val dataType = metricSchema match { + case _: CounterSchema => "counter" + case _: GaugeSchema => "gauge" + case _: HistogramSchema => "histogram" + } + jsonGenerator.writeStringField("kind", dataType) + jsonGenerator.writeObjectFieldStart("source") + jsonGenerator.writeStringField( + "class", + metricSchema.metricBuilder.sourceClass.getOrElse("Unspecified")) + jsonGenerator.writeStringField("category", metricSchema.metricBuilder.role.toString) + jsonGenerator.writeStringField( + "process_path", + metricSchema.metricBuilder.processPath.getOrElse("Unspecified")) + jsonGenerator.writeEndObject() + jsonGenerator.writeStringField("description", metricSchema.metricBuilder.description) + jsonGenerator.writeStringField("unit", metricSchema.metricBuilder.units.toString) + jsonGenerator.writeStringField("verbosity", metricSchema.metricBuilder.verbosity.toString) + jsonGenerator.writeBooleanField("key_indicator", metricSchema.metricBuilder.keyIndicator) + + if (metricSchema.isInstanceOf[HistogramSchema]) { + jsonGenerator.writeObjectFieldStart("buckets") + jsonGenerator.writeStringField("count", statsFormatter.separator + "count") + jsonGenerator.writeStringField("sum", statsFormatter.separator + "sum") + jsonGenerator.writeStringField( + "average", + statsFormatter.separator + statsFormatter.labelAverage) + jsonGenerator.writeStringField("minimum", statsFormatter.separator + statsFormatter.labelMin) + jsonGenerator.writeStringField("maximum", statsFormatter.separator + statsFormatter.labelMax) + metricSchema.metricBuilder.percentiles.foreach(bucket => + jsonGenerator.writeStringField( + bucket.toString, + statsFormatter.separator + statsFormatter.labelPercentile(bucket))) + + jsonGenerator.writeEndObject() + } + jsonGenerator.writeEndObject() + } + + private def convertNullToString(segment: String): String = { + if (segment == null) "null" + else segment + } +} diff --git a/server/src/main/scala/com/twitter/server/util/MetricSchemaSource.scala b/server/src/main/scala/com/twitter/server/util/MetricSchemaSource.scala index f278fb0d..076ce151 100644 --- a/server/src/main/scala/com/twitter/server/util/MetricSchemaSource.scala +++ b/server/src/main/scala/com/twitter/server/util/MetricSchemaSource.scala @@ -1,5 +1,6 @@ package com.twitter.server.util +import com.twitter.finagle.stats.exp.ExpressionSchema import com.twitter.finagle.stats.{MetricSchema, SchemaRegistry} import com.twitter.finagle.util.LoadService @@ -45,4 +46,8 @@ private[server] class MetricSchemaSource( registry .foldLeft(Set[String]()) { (set, r) => set ++ r.schemas().keySet } } + + def expressionList: Iterable[ExpressionSchema] = synchronized { + registry.foldLeft(IndexedSeq[ExpressionSchema]()) { (seq, r) => seq ++ r.expressions().values } + } } diff --git a/server/src/main/scala/com/twitter/server/util/exp/ExpressionSchemaSerializer.scala b/server/src/main/scala/com/twitter/server/util/exp/ExpressionSchemaSerializer.scala new file mode 100644 index 00000000..968cde36 --- /dev/null +++ b/server/src/main/scala/com/twitter/server/util/exp/ExpressionSchemaSerializer.scala @@ -0,0 +1,65 @@ +package com.twitter.server.util.exp + +import com.fasterxml.jackson.core.JsonGenerator +import com.fasterxml.jackson.databind.SerializerProvider +import com.fasterxml.jackson.databind.ser.std.StdSerializer +import com.twitter.finagle.stats.exp.{ + ConstantExpression, + Expression, + ExpressionSchema, + FunctionExpression, + MetricExpression +} +import com.twitter.finagle.stats.metadataScopeSeparator + +object ExpressionSchemaSerializer + extends StdSerializer[ExpressionSchema](classOf[ExpressionSchema]) { + + def serialize( + expressionSchema: ExpressionSchema, + gen: JsonGenerator, + provider: SerializerProvider + ): Unit = { + gen.writeStartObject() + gen.writeStringField("name", expressionSchema.name) + + gen.writeObjectFieldStart("labels") + gen.writeStringField( + "process_path", + expressionSchema.labels.processPath.getOrElse("Unspecified")) + gen.writeStringField( + "service_name", + expressionSchema.labels.serviceName.getOrElse("Unspecified")) + gen.writeStringField("role", expressionSchema.labels.role.toString) + gen.writeEndObject() + + gen.writeObjectFieldStart("expression") + writeExpression(expressionSchema.expr, gen) + gen.writeEndObject() + + gen.writeStringField("bounds", expressionSchema.bounds.toString) + gen.writeStringField("description", expressionSchema.description) + gen.writeStringField("unit", expressionSchema.unit.toString) + gen.writeEndObject() + } + + def writeExpression( + expr: Expression, + gen: JsonGenerator, + name: String = "metric" + ): Unit = { + expr match { + case MetricExpression(schema) => + gen.writeStringField(name, schema.metricBuilder.name.mkString(metadataScopeSeparator())) + case FunctionExpression(funcName, exprs) => + gen.writeStringField("operator", funcName) + gen.writeObjectFieldStart("metrics") + exprs.zipWithIndex.map { + case (expr, index) => writeExpression(expr, gen, name + "-" + index.toString) + } + gen.writeEndObject() + case ConstantExpression(repr, _) => + gen.writeStringField("constant", repr) + } + } +} diff --git a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala index 7cc34ca8..b2df4afc 100644 --- a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala @@ -2,6 +2,7 @@ package com.twitter.server.handler import com.twitter.finagle.http.Request import com.twitter.finagle.stats._ +import com.twitter.finagle.stats.exp.ExpressionSchema import com.twitter.server.util.MetricSchemaSource import com.twitter.util.Await import org.scalatest.FunSuite @@ -58,12 +59,14 @@ class MetricMetadataQueryHandlerTest extends FunSuite { trait UnlatchedRegistry extends SchemaRegistry { val hasLatchedCounters = false - override def schemas(): Map[String, MetricSchema] = schemaMap + def schemas(): Map[String, MetricSchema] = schemaMap + def expressions(): Map[String, ExpressionSchema] = Map.empty } trait LatchedRegistry extends SchemaRegistry { val hasLatchedCounters = true override def schemas(): Map[String, MetricSchema] = schemaMap + def expressions(): Map[String, ExpressionSchema] = Map.empty } val latchedSchemaRegistry = new LatchedRegistry() {} diff --git a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala new file mode 100644 index 00000000..6695bb27 --- /dev/null +++ b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala @@ -0,0 +1,123 @@ +package com.twitter.server.handler.exp + +import com.twitter.finagle.http.Request +import com.twitter.finagle.stats.exp.{Expression, ExpressionSchema} +import com.twitter.finagle.stats.{ + CounterSchema, + HistogramSchema, + InMemoryStatsReceiver, + MetricBuilder, + MetricSchema, + SchemaRegistry +} +import com.twitter.server.handler.MetricExpressionHandler +import com.twitter.server.util.MetricSchemaSource +import com.twitter.util.Await +import org.scalatest.FunSuite + +class MetricExpressionHandlerTest extends FunSuite { + + val sr = new InMemoryStatsReceiver + + val successMb = CounterSchema(new MetricBuilder(name = Seq("success"), statsReceiver = sr)) + val failuresMb = + CounterSchema(new MetricBuilder(name = Seq("failures"), statsReceiver = sr)) + val latencyMb = HistogramSchema(new MetricBuilder(name = Seq("latency"), statsReceiver = sr)) + + val successRateExpression = ExpressionSchema( + "success_rate", + Expression(successMb).divide(Expression(successMb).plus(Expression(failuresMb)))) + val throughputExpression = + ExpressionSchema("throughput", Expression(successMb).plus(Expression(failuresMb))) + val latencyExpression = ExpressionSchema("latency", Expression(latencyMb)) + + val expressionSchemaMap: Map[String, ExpressionSchema] = Map( + "success_rate" -> successRateExpression, + "throughput" -> throughputExpression, + "latency" -> latencyExpression + ) + + val expressionRegistry = new SchemaRegistry { + def hasLatchedCounters: Boolean = false + def schemas(): Map[String, MetricSchema] = Map.empty + val expressions: Map[String, ExpressionSchema] = expressionSchemaMap + } + val expressionSource = new MetricSchemaSource(Seq(expressionRegistry)) + + val expressionHandler = new MetricExpressionHandler(expressionSource) + + val testRequest = Request("http://$HOST:$PORT/admin/metric/expressions.json") + + private[this] def assertJsonResponse(actual: String, expected: String) = { + assert(stripWhitespace(actual) == stripWhitespace(expected)) + } + + private[this] def stripWhitespace(string: String): String = + string.filter { case c => !c.isWhitespace } + + test("Get the all expressions") { + val responseString = + """ + |{ + | "@version" : 0.1, + | "expressions" : [ + | { + | "name" : "success_rate", + | "labels" : { + | "process_path" : "Unspecified", + | "service_name" : "Unspecified", + | "role" : "NoRoleSpecified" + | }, + | "expression" : { + | "operator" : "divide", + | "metrics" : { + | "metric-0" : "success", + | "operator" : "plus", + | "metrics" : { + | "metric-1-0" : "success", + | "metric-1-1" : "failures" + | } + | } + | }, + | "bounds" : "Unbounded", + | "description" : "Unspecified", + | "unit" : "Unspecified" + | }, + | { + | "name" : "throughput", + | "labels" : { + | "process_path" : "Unspecified", + | "service_name" : "Unspecified", + | "role" : "NoRoleSpecified" + | }, + | "expression" : { + | "operator" : "plus", + | "metrics" : { + | "metric-0" : "success", + | "metric-1" : "failures" + | } + | }, + | "bounds" : "Unbounded", + | "description" : "Unspecified", + | "unit" : "Unspecified" + | }, + | { + | "name" : "latency", + | "labels" : { + | "process_path" : "Unspecified", + | "service_name" : "Unspecified", + | "role" : "NoRoleSpecified" + | }, + | "expression" : { + | "metric" : "latency" + | }, + | "bounds" : "Unbounded", + | "description" : "Unspecified", + | "unit" : "Unspecified" + | } + | ] + |}""".stripMargin + + assertJsonResponse(responseString, Await.result(expressionHandler(testRequest)).contentString) + } +} diff --git a/server/src/test/scala/com/twitter/server/util/MetricSchemaSourceTest.scala b/server/src/test/scala/com/twitter/server/util/MetricSchemaSourceTest.scala index 27efbe79..4333ac60 100644 --- a/server/src/test/scala/com/twitter/server/util/MetricSchemaSourceTest.scala +++ b/server/src/test/scala/com/twitter/server/util/MetricSchemaSourceTest.scala @@ -1,6 +1,7 @@ package com.twitter.server.util import com.twitter.finagle.stats._ +import com.twitter.finagle.stats.exp.ExpressionSchema import org.scalatest.FunSuite class MetricSchemaSourceTest extends FunSuite { @@ -41,13 +42,15 @@ class MetricSchemaSourceTest extends FunSuite { ) private val latchedPopulatedRegistry: SchemaRegistry = new SchemaRegistry { - override def hasLatchedCounters: Boolean = true - override def schemas(): Map[String, MetricSchema] = schemaMap + def hasLatchedCounters: Boolean = true + def schemas(): Map[String, MetricSchema] = schemaMap + def expressions(): Map[String, ExpressionSchema] = Map.empty } private val unlatchedPopulatedRegistry: SchemaRegistry = new SchemaRegistry { - override def hasLatchedCounters: Boolean = false - override def schemas(): Map[String, MetricSchema] = schemaMap + def hasLatchedCounters: Boolean = false + def schemas(): Map[String, MetricSchema] = schemaMap + def expressions(): Map[String, ExpressionSchema] = Map.empty } private val metricSchemaSource = new MetricSchemaSource(Seq(latchedPopulatedRegistry)) From a9340873b3231357ef308620cf4bac68ea0f7124 Mon Sep 17 00:00:00 2001 From: Joy Bestourous Date: Fri, 12 Mar 2021 19:46:09 +0000 Subject: [PATCH 229/409] {finagle, finatra, util, twitter-server, scrooge}: Drop scala 2.11 for OSS Problem/Solution: Two scala versions (2.12, 2.13) is sufficient. Let's drop scala 2.11. JIRA Issues: CSL-10599 Differential Revision: https://phabricator.twitter.biz/D631091 --- .github/workflows/continuous-integration.yml | 2 +- CHANGELOG.rst | 2 ++ build.sbt | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 108e0605..9b71d478 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -19,7 +19,7 @@ jobs: strategy: fail-fast: false matrix: - scala: [2.11.12, 2.12.8, 2.13.1] + scala: [2.12.8, 2.13.1] java: ['1.8', '1.11'] runs-on: ubuntu-latest steps: diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3ebdc156..d2a25f6d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,8 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +* Builds are now only supported for Scala 2.12+ ``PHAB_ID=D631091`` + 21.2.0 ------ diff --git a/build.sbt b/build.sbt index 4ca1be9a..d929dc08 100644 --- a/build.sbt +++ b/build.sbt @@ -77,7 +77,7 @@ lazy val sharedSettings = Seq( version := releaseVersion, organization := "com.twitter", scalaVersion := "2.12.11", - crossScalaVersions := Seq("2.11.12", "2.12.11", "2.13.1"), + crossScalaVersions := Seq("2.12.11", "2.13.1"), fork in Test := true, // We have to fork to get the JavaOptions libraryDependencies ++= Seq( // See https://www.scala-sbt.org/0.13/docs/Testing.html#JUnit From 8d6cc59de1270be5f957d577193597eceb2639e5 Mon Sep 17 00:00:00 2001 From: Adam Singer Date: Tue, 16 Mar 2021 06:18:44 +0000 Subject: [PATCH 230/409] Bump scala version to 2.12.13 Problem/Solution Bumping scala version to 2.12.13 JIRA Issues: SCALA-25 Differential Revision: https://phabricator.twitter.biz/D632567 --- CHANGELOG.rst | 5 +++++ build.sbt | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d2a25f6d..f6a97319 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Runtime Behavior Changes +~~~~~~~~~~~~~~~~~~~~~~~~ + +* Bump scala version to 2.12.13 ``PHAB_ID=D632567`` + * Builds are now only supported for Scala 2.12+ ``PHAB_ID=D631091`` 21.2.0 diff --git a/build.sbt b/build.sbt index d929dc08..2febe6d6 100644 --- a/build.sbt +++ b/build.sbt @@ -76,8 +76,8 @@ def travisTestJavaOptions: Seq[String] = { lazy val sharedSettings = Seq( version := releaseVersion, organization := "com.twitter", - scalaVersion := "2.12.11", - crossScalaVersions := Seq("2.12.11", "2.13.1"), + scalaVersion := "2.12.13", + crossScalaVersions := Seq("2.12.13", "2.13.1"), fork in Test := true, // We have to fork to get the JavaOptions libraryDependencies ++= Seq( // See https://www.scala-sbt.org/0.13/docs/Testing.html#JUnit From 2c549043d6ef70ac54ad4c472f5fb2349904d05b Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Tue, 16 Mar 2021 19:26:47 +0000 Subject: [PATCH 231/409] Bump scala version to 2.12.13 in workflows Problem/Solution We've updated the supported scala version in the sbt files lets follow up with the CI workflows. Util was already done in the first update. Differential Revision: https://phabricator.twitter.biz/D635494 --- .github/workflows/continuous-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 9b71d478..28ed5018 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -19,7 +19,7 @@ jobs: strategy: fail-fast: false matrix: - scala: [2.12.8, 2.13.1] + scala: [2.12.13, 2.13.1] java: ['1.8', '1.11'] runs-on: ubuntu-latest steps: From a1a97d3009b457c63e2a2aa3bdc728dbb57d4172 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Tue, 16 Mar 2021 19:28:31 +0000 Subject: [PATCH 232/409] twitter-server: Add the current metric name as a url fragment in the metrics view Problem The metrics Watch endpoint will use the url fragment to go right to the metrics you're interested in which is super convenient. However, you essentially have to type this in by hand since it's never populated in the browsers url bar. Solution Set the url fragment when we start to watch a metric. This makes copying and sharing admin metrics pages super quick. Differential Revision: https://phabricator.twitter.biz/D635343 --- server/src/main/resources/twitter-server/js/metric-query.js | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/resources/twitter-server/js/metric-query.js b/server/src/main/resources/twitter-server/js/metric-query.js index 48728083..d4031e3f 100644 --- a/server/src/main/resources/twitter-server/js/metric-query.js +++ b/server/src/main/resources/twitter-server/js/metric-query.js @@ -27,6 +27,7 @@ function graphLibLoaded() { if (selected != undefined) selected.removeClass("selected") li.addClass("selected") selected = li + window.location.hash = "#" + stat refreshStats(stat, new ChartRenderer($("#chart-div")[0], stat)) } From 869b18639246f1995682dcd734581320db462943 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Wed, 17 Mar 2021 01:35:50 +0000 Subject: [PATCH 233/409] Revert to scala 2.12.12 due to scoverage issue with scala 2.12.13 Problem/Solution The scoverage plugin has an issue due to a binary incompatibility introduced in scala 2.12.13 (see: https://github.com/scoverage/sbt-scoverage/issues/319). It has been patched in the plugin in https://github.com/scoverage/scalac-scoverage-plugin/pull/306 which is currently awaiting a release. However after updating our projects to scala 2.12.13, coverage is broken and failing the CI builds. Downgrade to scala 2.12.12 until the scoverage plugin has been published with the fix. Differential Revision: https://phabricator.twitter.biz/D635917 --- .github/workflows/continuous-integration.yml | 2 +- CHANGELOG.rst | 3 +++ build.sbt | 4 ++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 28ed5018..8fa76e7d 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -19,7 +19,7 @@ jobs: strategy: fail-fast: false matrix: - scala: [2.12.13, 2.13.1] + scala: [2.12.12, 2.13.1] java: ['1.8', '1.11'] runs-on: ubuntu-latest steps: diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f6a97319..b0765a7d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,9 @@ Unreleased Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ +* Revert to scala version 2.12.12 due to https://github.com/scoverage/sbt-scoverage/issues/319 + ``PHAB_ID=D635917`` + * Bump scala version to 2.12.13 ``PHAB_ID=D632567`` * Builds are now only supported for Scala 2.12+ ``PHAB_ID=D631091`` diff --git a/build.sbt b/build.sbt index 2febe6d6..89e2fa1f 100644 --- a/build.sbt +++ b/build.sbt @@ -76,8 +76,8 @@ def travisTestJavaOptions: Seq[String] = { lazy val sharedSettings = Seq( version := releaseVersion, organization := "com.twitter", - scalaVersion := "2.12.13", - crossScalaVersions := Seq("2.12.13", "2.13.1"), + scalaVersion := "2.12.12", + crossScalaVersions := Seq("2.12.12", "2.13.1"), fork in Test := true, // We have to fork to get the JavaOptions libraryDependencies ++= Seq( // See https://www.scala-sbt.org/0.13/docs/Testing.html#JUnit From ab4f57038a0c14fbfabb1061f49fe1794f8e934a Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Thu, 18 Mar 2021 19:53:24 +0000 Subject: [PATCH 234/409] twitter/oss: Unset TRAVIS env var Problem We set the `TRAVIS` env var to convince the various `build.sbt` files to set appropriate configuration in the forked test processes. However, this is confusing our code coverage tool as it is picking up the env var and attempting to verify the project as a TRAVIS CI build, which it is not and thus coverage uploading fails. Solution Unset the `TRAVIS` env var before uploading results such that the auto detection of the CI provider properly picks up the GitHub Action env vars and does the right thing. Differential Revision: https://phabricator.twitter.biz/D636932 --- .github/workflows/continuous-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 8fa76e7d..acb89355 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -65,4 +65,4 @@ jobs: run: ${{ format('./sbt ++{0} coverageAggregate', matrix.scala) }} - name: upload coverage if: ${{ success() }} - run: bash <(curl -s https://codecov.io/bash) + run: unset TRAVIS; bash <(curl -s https://codecov.io/bash) From c90fa00caebab901ac786e320de681bca43447c8 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Fri, 19 Mar 2021 21:10:07 +0000 Subject: [PATCH 235/409] twitter-server: Expression thresholds to JSON Problem/Solution This revision's goal is serializing and deserializing Expression Bounds (thresholds) to/from JSON nicely. The Bounds are self-explained case classes that can leverage Jackson Json mapping, some special cases such as deserializing Operator is supported by adding a customized deserializer. Two additional changes that made for supporting this are: ``` 1. Set ObjectMapper to use snake case naming strategy, all of our customized de/serializers are writing snake case, make the object mapper also snake case to be consistent. The JsonConverter has a few usages outside of the twitter-server, we keep that as a utility helper and switch the twitter-server Admin endpoints to use the configured one. 2. Some Json utilities are redundant. merge them. ``` JIRA Issues: CSL-10619, CSL-10803 Differential Revision: https://phabricator.twitter.biz/D634550 --- CHANGELOG.rst | 11 +++ .../server/handler/AdminHttpMuxHandler.scala | 2 +- .../server/handler/AnnouncerHandler.scala | 4 +- .../handler/AttachedClientsHandler.scala | 6 +- .../twitter/server/handler/DtabHandler.scala | 4 +- .../handler/FailedLintRuleHandler.scala | 2 +- .../server/handler/HeapResourceHandler.scala | 4 +- .../handler/HistogramQueryHandler.scala | 8 +- .../twitter/server/handler/LintHandler.scala | 8 +- .../handler/MetricExpressionHandler.scala | 19 ++-- .../handler/MetricMetadataQueryHandler.scala | 4 +- .../server/handler/MetricQueryHandler.scala | 4 +- .../handler/MetricTypeQueryHandler.scala | 4 +- .../handler/ProfileResourceHandler.scala | 4 +- .../server/handler/RegistryHandler.scala | 4 +- .../server/handler/ServerInfoHandler.scala | 4 +- .../server/handler/ShutdownHandler.scala | 4 +- .../server/handler/ThreadsHandler.scala | 4 +- .../server/handler/ToggleHandler.scala | 6 +- .../server/handler/TunableHandler.scala | 10 +-- .../scala/com/twitter/server/util/Json.scala | 72 ++++++++++++--- .../twitter/server/util/JsonConverter.scala | 33 ------- .../server/util/MetricSchemaJsonModule.scala | 4 +- .../server/util/exp/ExpressionJson.scala | 90 +++++++++++++++++++ .../util/exp/ExpressionSchemaSerializer.scala | 65 -------------- .../server/view/BalancersJsonView.scala | 4 +- .../server/HistogramQueryHandlerTest.scala | 11 +-- .../MetricMetadataQueryHandlerTest.scala | 16 +--- .../handler/exp/ExpressionSerdeTest.scala | 83 +++++++++++++++++ .../exp/MetricExpressionHandlerTest.scala | 47 ++++++---- .../com/twitter/server/util/JsonUtils.scala | 17 ++++ .../util/MetricSchemaJsonModuleTest.scala | 6 +- 32 files changed, 366 insertions(+), 198 deletions(-) delete mode 100644 server/src/main/scala/com/twitter/server/util/JsonConverter.scala create mode 100644 server/src/main/scala/com/twitter/server/util/exp/ExpressionJson.scala delete mode 100644 server/src/main/scala/com/twitter/server/util/exp/ExpressionSchemaSerializer.scala create mode 100644 server/src/test/scala/com/twitter/server/handler/exp/ExpressionSerdeTest.scala create mode 100644 server/src/test/scala/com/twitter/server/util/JsonUtils.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b0765a7d..3ba2a5e2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,17 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Breaking API Changes +~~~~~~~~~~~~~~~~~~~~ + +* com.twitter.server.util.JsonConverter is divided into two Json mappers, JsonConverter + and AdminJsonConverter. JsonConverter keeps the minimum configuration, for json mapping + to/from twitter-server admin endpoints, AdminJsonConverter is configured to do so. + ``PHAB_ID=D634550`` + +* Json.scala and JsonConverter are merged, removed the Json.deserialize() methods. + ``PHAB_ID=D634550`` + Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/server/src/main/scala/com/twitter/server/handler/AdminHttpMuxHandler.scala b/server/src/main/scala/com/twitter/server/handler/AdminHttpMuxHandler.scala index 7e7180f7..a1824a30 100644 --- a/server/src/main/scala/com/twitter/server/handler/AdminHttpMuxHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/AdminHttpMuxHandler.scala @@ -1,7 +1,7 @@ package com.twitter.server.handler import com.twitter.finagle.Service -import com.twitter.finagle.http.{Route, Request, Response} +import com.twitter.finagle.http.{Request, Response, Route} /** * Trait AdminHttpMuxHandler is used for service-loading HTTP handlers specifically for the AdminHttpServer diff --git a/server/src/main/scala/com/twitter/server/handler/AnnouncerHandler.scala b/server/src/main/scala/com/twitter/server/handler/AnnouncerHandler.scala index 000ee165..1d00a2cc 100644 --- a/server/src/main/scala/com/twitter/server/handler/AnnouncerHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/AnnouncerHandler.scala @@ -1,9 +1,9 @@ package com.twitter.server.handler -import com.twitter.finagle.{Announcer, Service} import com.twitter.finagle.http.{Request, Response} -import com.twitter.util.Future +import com.twitter.finagle.{Announcer, Service} import com.twitter.server.util.HttpUtils.newOk +import com.twitter.util.Future class AnnouncerHandler extends Service[Request, Response] { def apply(req: Request): Future[Response] = { diff --git a/server/src/main/scala/com/twitter/server/handler/AttachedClientsHandler.scala b/server/src/main/scala/com/twitter/server/handler/AttachedClientsHandler.scala index 1de48ca5..5b4759a1 100644 --- a/server/src/main/scala/com/twitter/server/handler/AttachedClientsHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/AttachedClientsHandler.scala @@ -1,13 +1,13 @@ package com.twitter.server.handler -import com.fasterxml.jackson.annotation.{JsonFormat, JsonInclude} import com.fasterxml.jackson.annotation.JsonInclude.Include +import com.fasterxml.jackson.annotation.{JsonFormat, JsonInclude} import com.fasterxml.jackson.databind.PropertyNamingStrategy import com.fasterxml.jackson.databind.annotation.JsonNaming import com.twitter.finagle.Service import com.twitter.finagle.http.{Request, Response} import com.twitter.finagle.server.ServerRegistry -import com.twitter.server.util.JsonConverter +import com.twitter.server.util.AdminJsonConverter import com.twitter.util.Future import java.net.SocketAddress import java.security.cert.X509Certificate @@ -95,6 +95,6 @@ class AttachedClientsHandler(serverRegistry: ServerRegistry = ServerRegistry) extends Service[Request, Response] { def apply(request: Request): Future[Response] = { val doc = AttachedClientsHandler.render(serverRegistry) - Future.value(JsonConverter(doc)) + Future.value(AdminJsonConverter(doc)) } } diff --git a/server/src/main/scala/com/twitter/server/handler/DtabHandler.scala b/server/src/main/scala/com/twitter/server/handler/DtabHandler.scala index f62bd9be..1c1a3cd5 100644 --- a/server/src/main/scala/com/twitter/server/handler/DtabHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/DtabHandler.scala @@ -3,7 +3,7 @@ package com.twitter.server.handler import com.twitter.finagle.http.{MediaType, Request, Response} import com.twitter.finagle.{Dtab, Service} import com.twitter.io.Buf -import com.twitter.server.util.{HttpUtils, JsonConverter} +import com.twitter.server.util.{AdminJsonConverter, HttpUtils} import com.twitter.util.Future /** @@ -31,7 +31,7 @@ final class DtabHandler extends Service[Request, Response] { HttpUtils.newResponse(contentType = MediaType.Json, content = Buf.Utf8(jsonResponse)) private[this] def jsonResponse: String = { - JsonConverter.writeToString(Map("dtab" -> formattedDtabEntries)) + AdminJsonConverter.writeToString(Map("dtab" -> formattedDtabEntries)) } private[this] def formattedDtabEntries: Seq[String] = { diff --git a/server/src/main/scala/com/twitter/server/handler/FailedLintRuleHandler.scala b/server/src/main/scala/com/twitter/server/handler/FailedLintRuleHandler.scala index 4179c709..0801f11b 100644 --- a/server/src/main/scala/com/twitter/server/handler/FailedLintRuleHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/FailedLintRuleHandler.scala @@ -5,8 +5,8 @@ import com.twitter.finagle.Service import com.twitter.finagle.http.{Request, Response} import com.twitter.io.Buf import com.twitter.server.util.HttpUtils.newResponse -import com.twitter.util.{Future, Time} import com.twitter.util.lint.{GlobalRules, Rule} +import com.twitter.util.{Future, Time} /** * Renders failed lint rule alert in an html fragment onto /admin/failedlint. diff --git a/server/src/main/scala/com/twitter/server/handler/HeapResourceHandler.scala b/server/src/main/scala/com/twitter/server/handler/HeapResourceHandler.scala index 0f297768..7f148ae5 100644 --- a/server/src/main/scala/com/twitter/server/handler/HeapResourceHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/HeapResourceHandler.scala @@ -1,13 +1,13 @@ package com.twitter.server.handler import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.http.{Request, Response, Status, Uri} import com.twitter.finagle.Service +import com.twitter.finagle.http.{Request, Response, Status, Uri} import com.twitter.io.Buf import com.twitter.jvm.Heapster import com.twitter.server.util.HttpUtils.newResponse -import com.twitter.util.{Duration, Future} import com.twitter.util.logging.Logger +import com.twitter.util.{Duration, Future} class HeapResourceHandler extends Service[Request, Response] { private[this] val log = Logger[HeapResourceHandler] diff --git a/server/src/main/scala/com/twitter/server/handler/HistogramQueryHandler.scala b/server/src/main/scala/com/twitter/server/handler/HistogramQueryHandler.scala index 951c255a..a99910c9 100644 --- a/server/src/main/scala/com/twitter/server/handler/HistogramQueryHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/HistogramQueryHandler.scala @@ -4,9 +4,9 @@ import com.twitter.finagle.Service import com.twitter.finagle.http.{Request, Response, Uri} import com.twitter.finagle.stats.{BucketAndCount, HistogramDetail, WithHistogramDetails} import com.twitter.io.Buf +import com.twitter.server.util.AdminJsonConverter import com.twitter.server.util.HtmlUtils.escapeHtml import com.twitter.server.util.HttpUtils.newResponse -import com.twitter.server.util.JsonConverter import com.twitter.util.Future object HistogramQueryHandler { @@ -65,7 +65,7 @@ object HistogramQueryHandler { ): String = // ".toMap" is important here for scala 2.13 as otherwise it will be a MapView which // doesn't serialize correctly with Jackson - JsonConverter.writeToString(counts.mapValues(transform).toMap) + AdminJsonConverter.writeToString(counts.mapValues(transform).toMap) // Generates html for visualizing histograms private[HistogramQueryHandler] val render: String = { @@ -205,7 +205,7 @@ private[server] class HistogramQueryHandler(details: WithHistogramDetails) ) private[this] def renderHistogramsJson: String = - JsonConverter.writeToString(histograms.map { + AdminJsonConverter.writeToString(histograms.map { case (key, value) => (key, value.counts) }) @@ -271,7 +271,7 @@ private[server] class HistogramQueryHandler(details: WithHistogramDetails) } private[this] def renderSummary(summary: Summary): String = - JsonConverter.writeToString(summary) + AdminJsonConverter.writeToString(summary) private[this] def htmlResponse(query: String): Future[Response] = newResponse( diff --git a/server/src/main/scala/com/twitter/server/handler/LintHandler.scala b/server/src/main/scala/com/twitter/server/handler/LintHandler.scala index 9a0874f4..eed96b1e 100644 --- a/server/src/main/scala/com/twitter/server/handler/LintHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/LintHandler.scala @@ -1,14 +1,14 @@ package com.twitter.server.handler import com.twitter.finagle.Service -import com.twitter.finagle.http.{Response, Request} +import com.twitter.finagle.http.{Request, Response} import com.twitter.io.Buf import com.twitter.server.handler.LintHandler.LintView +import com.twitter.server.util.AdminJsonConverter import com.twitter.server.util.HtmlUtils.escapeHtml import com.twitter.server.util.HttpUtils._ -import com.twitter.server.util.JsonConverter import com.twitter.util.Future -import com.twitter.util.lint.{Issue, Rule, GlobalRules} +import com.twitter.util.lint.{GlobalRules, Issue, Rule} /** * UI for running the globally registered lint [[Rule Rules]]. @@ -49,7 +49,7 @@ class LintHandler extends Service[Request, Response] { private[this] def jsonResponse(req: Request): Future[Response] = { val rules = GlobalRules.get.iterable.toSeq val jsonMap = jsonMapFromRules(rules) - newOk(JsonConverter.writeToString(jsonMap), "application/json;charset=UTF-8") + newOk(AdminJsonConverter.writeToString(jsonMap), "application/json;charset=UTF-8") } /** exposed for testing */ diff --git a/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala b/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala index a61ec7c5..1f5dacb7 100644 --- a/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala @@ -2,22 +2,31 @@ package com.twitter.server.handler import com.twitter.finagle.Service import com.twitter.finagle.http.{MediaType, Request, Response} +import com.twitter.finagle.stats.metadataScopeSeparator import com.twitter.io.Buf +import com.twitter.server.handler.MetricExpressionHandler.Version import com.twitter.server.util.HttpUtils.newResponse -import com.twitter.server.util.{JsonConverter, MetricSchemaSource} +import com.twitter.server.util.{AdminJsonConverter, MetricSchemaSource} import com.twitter.util.Future +object MetricExpressionHandler { + private val Version = 0.2 +} + class MetricExpressionHandler(source: MetricSchemaSource = new MetricSchemaSource) extends Service[Request, Response] { def apply(request: Request): Future[Response] = { - val expressions = source.expressionList newResponse( contentType = MediaType.JsonUtf8, content = Buf.Utf8( - JsonConverter.writeToString( - Map("@version" -> 0.1, "expressions" -> expressions) - ) + AdminJsonConverter.writeToString( + Map( + "@version" -> Version, + "counters_latched" -> source.hasLatchedCounters, + "separator_char" -> metadataScopeSeparator(), + "expressions" -> source.expressionList + )) ) ) } diff --git a/server/src/main/scala/com/twitter/server/handler/MetricMetadataQueryHandler.scala b/server/src/main/scala/com/twitter/server/handler/MetricMetadataQueryHandler.scala index 717c7878..24d99bde 100644 --- a/server/src/main/scala/com/twitter/server/handler/MetricMetadataQueryHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/MetricMetadataQueryHandler.scala @@ -5,7 +5,7 @@ import com.twitter.finagle.http.{MediaType, Request, Response, Uri} import com.twitter.finagle.stats.metadataScopeSeparator import com.twitter.io.Buf import com.twitter.server.util.HttpUtils.newResponse -import com.twitter.server.util.{JsonConverter, MetricSchemaSource} +import com.twitter.server.util.{AdminJsonConverter, MetricSchemaSource} import com.twitter.util.Future /** @@ -100,7 +100,7 @@ class MetricMetadataQueryHandler(source: MetricSchemaSource = new MetricSchemaSo newResponse( contentType = MediaType.JsonUtf8, content = Buf.Utf8( - JsonConverter.writeToString( + AdminJsonConverter.writeToString( Map( "@version" -> 3.0, "counters_latched" -> latched, diff --git a/server/src/main/scala/com/twitter/server/handler/MetricQueryHandler.scala b/server/src/main/scala/com/twitter/server/handler/MetricQueryHandler.scala index 2007eb7b..33a2fae5 100644 --- a/server/src/main/scala/com/twitter/server/handler/MetricQueryHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/MetricQueryHandler.scala @@ -5,7 +5,7 @@ import com.twitter.finagle.http.{MediaType, Request, Response, Uri} import com.twitter.io.Buf import com.twitter.server.util.HtmlUtils.escapeHtml import com.twitter.server.util.HttpUtils.newResponse -import com.twitter.server.util.{JsonConverter, MetricSource} +import com.twitter.server.util.{AdminJsonConverter, MetricSource} import com.twitter.util.Future private object MetricQueryHandler { @@ -45,7 +45,7 @@ class MetricQueryHandler(source: MetricSource = new MetricSource) val someKeys = uri.params.getAll("m") newResponse( contentType = MediaType.JsonUtf8, - content = Buf.Utf8(JsonConverter.writeToString(query(someKeys))) + content = Buf.Utf8(AdminJsonConverter.writeToString(query(someKeys))) ) } else { newResponse( diff --git a/server/src/main/scala/com/twitter/server/handler/MetricTypeQueryHandler.scala b/server/src/main/scala/com/twitter/server/handler/MetricTypeQueryHandler.scala index 8ecb2d9f..e01e302c 100644 --- a/server/src/main/scala/com/twitter/server/handler/MetricTypeQueryHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/MetricTypeQueryHandler.scala @@ -5,7 +5,7 @@ import com.twitter.finagle.http.{MediaType, Request, Response, Uri} import com.twitter.finagle.stats.{HistogramDetail, WithHistogramDetails} import com.twitter.io.Buf import com.twitter.server.util.HttpUtils.newResponse -import com.twitter.server.util.{JsonConverter, MetricSource} +import com.twitter.server.util.{AdminJsonConverter, MetricSource} import com.twitter.util.Future /** @@ -78,7 +78,7 @@ class MetricTypeQueryHandler( newResponse( contentType = MediaType.JsonUtf8, content = Buf.Utf8( - JsonConverter.writeToString( + AdminJsonConverter.writeToString( Map( "latched" -> latched, "metrics" -> metrics diff --git a/server/src/main/scala/com/twitter/server/handler/ProfileResourceHandler.scala b/server/src/main/scala/com/twitter/server/handler/ProfileResourceHandler.scala index f19b9986..5a71541e 100644 --- a/server/src/main/scala/com/twitter/server/handler/ProfileResourceHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/ProfileResourceHandler.scala @@ -1,14 +1,14 @@ package com.twitter.server.handler import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.http.{Request, Response, Status, Uri} import com.twitter.finagle.Service +import com.twitter.finagle.http.{Request, Response, Status, Uri} import com.twitter.io.Buf import com.twitter.jvm.CpuProfile import com.twitter.server.util.HttpUtils.newResponse +import com.twitter.util.logging.Logger import com.twitter.util.{Duration, Future, Return, Throw} import java.io.ByteArrayOutputStream -import com.twitter.util.logging.Logger class ProfileResourceHandler(which: Thread.State) extends Service[Request, Response] { private[this] val log = Logger[ProfileResourceHandler] diff --git a/server/src/main/scala/com/twitter/server/handler/RegistryHandler.scala b/server/src/main/scala/com/twitter/server/handler/RegistryHandler.scala index b1eacba7..cf52d9e5 100644 --- a/server/src/main/scala/com/twitter/server/handler/RegistryHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/RegistryHandler.scala @@ -2,8 +2,8 @@ package com.twitter.server.handler import com.twitter.finagle.Service import com.twitter.finagle.http.{Request, Response} +import com.twitter.server.util.AdminJsonConverter import com.twitter.server.util.HttpUtils.newOk -import com.twitter.server.util.JsonConverter import com.twitter.util.Future import com.twitter.util.registry.{Formatter, GlobalRegistry, Registry, SimpleRegistry} @@ -76,7 +76,7 @@ class RegistryHandler extends Service[Request, Response] { private[handler] def jsonResponse(filter: Option[String]): String = { val filtered: Registry = filterRegistry(filter) val asMap: Map[String, Object] = Formatter.asMap(filtered) - JsonConverter.writeToString(asMap) + AdminJsonConverter.writeToString(asMap) } } diff --git a/server/src/main/scala/com/twitter/server/handler/ServerInfoHandler.scala b/server/src/main/scala/com/twitter/server/handler/ServerInfoHandler.scala index bb3d82c9..6da4cd58 100644 --- a/server/src/main/scala/com/twitter/server/handler/ServerInfoHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/ServerInfoHandler.scala @@ -5,8 +5,8 @@ import com.twitter.finagle.http.{Request, Response} import com.twitter.finagle.stats.LoadedStatsReceiver import com.twitter.io.Buf import com.twitter.server.BuildProperties +import com.twitter.server.util.AdminJsonConverter import com.twitter.server.util.HttpUtils.newResponse -import com.twitter.server.util.JsonConverter import com.twitter.util.Future import com.twitter.util.registry.GlobalRegistry import java.lang.management.ManagementFactory @@ -68,7 +68,7 @@ class ServerInfoHandler() extends Service[Request, Response] { newResponse( contentType = "application/json;charset=UTF-8", content = - Buf.Utf8(JsonConverter.writeToString(serverInfo + ("uptime" -> mxRuntime.getUptime))) + Buf.Utf8(AdminJsonConverter.writeToString(serverInfo + ("uptime" -> mxRuntime.getUptime))) ) } } diff --git a/server/src/main/scala/com/twitter/server/handler/ShutdownHandler.scala b/server/src/main/scala/com/twitter/server/handler/ShutdownHandler.scala index 307c2b1d..0108611f 100644 --- a/server/src/main/scala/com/twitter/server/handler/ShutdownHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/ShutdownHandler.scala @@ -1,12 +1,12 @@ package com.twitter.server.handler import com.twitter.app.App -import com.twitter.finagle.http.{Method, Request, Response, Status, Uri} import com.twitter.finagle.Service +import com.twitter.finagle.http.{Method, Request, Response, Status, Uri} import com.twitter.io.Buf import com.twitter.server.util.HttpUtils.{newOk, newResponse} -import com.twitter.util.{Duration, Future} import com.twitter.util.logging.Logger +import com.twitter.util.{Duration, Future} class ShutdownHandler(app: App) extends Service[Request, Response] { private[this] val log = Logger[ShutdownHandler] diff --git a/server/src/main/scala/com/twitter/server/handler/ThreadsHandler.scala b/server/src/main/scala/com/twitter/server/handler/ThreadsHandler.scala index 83b455d4..21dd38f9 100644 --- a/server/src/main/scala/com/twitter/server/handler/ThreadsHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/ThreadsHandler.scala @@ -3,8 +3,8 @@ package com.twitter.server.handler import com.twitter.finagle.Service import com.twitter.finagle.http.{Request, Response} import com.twitter.io.Buf +import com.twitter.server.util.AdminJsonConverter import com.twitter.server.util.HttpUtils.{expectsHtml, expectsJson, newOk, newResponse} -import com.twitter.server.util.JsonConverter import com.twitter.server.view.ThreadsView import com.twitter.util.Future import java.lang.management.ManagementFactory @@ -53,7 +53,7 @@ class ThreadsHandler extends Service[Request, Response] { ) } val msg = Map("threads" -> stacks) - newOk(JsonConverter.writeToString(msg), "application/json;charset=UTF-8") + newOk(AdminJsonConverter.writeToString(msg), "application/json;charset=UTF-8") } private def htmlResponse(req: Request): Future[Response] = { diff --git a/server/src/main/scala/com/twitter/server/handler/ToggleHandler.scala b/server/src/main/scala/com/twitter/server/handler/ToggleHandler.scala index f9b32cb2..7c67ae2b 100644 --- a/server/src/main/scala/com/twitter/server/handler/ToggleHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/ToggleHandler.scala @@ -4,8 +4,8 @@ import com.twitter.finagle.Service import com.twitter.finagle.http.{Fields, Method, Request, Response, Status} import com.twitter.finagle.toggle.{StandardToggleMap, Toggle, ToggleMap} import com.twitter.io.Buf +import com.twitter.server.util.AdminJsonConverter import com.twitter.server.util.HttpUtils.{newOk, newResponse} -import com.twitter.server.util.JsonConverter import com.twitter.util.Future import com.twitter.util.logging.Logger import scala.collection.mutable @@ -188,7 +188,7 @@ class ToggleHandler private[handler] (registeredLibrariesFn: () => Map[String, T headers: Iterable[(String, Object)] = Seq.empty ): Future[Response] = { // sort the errors to make the response more deterministic. - val body = JsonConverter.writeToString(GenericResponse(msg, errors.sorted)) + val body = AdminJsonConverter.writeToString(GenericResponse(msg, errors.sorted)) newResponse( status = status, contentType = "text/plain;charset=UTF-8", @@ -281,7 +281,7 @@ class ToggleHandler private[handler] (registeredLibrariesFn: () => Map[String, T /** package protected for testing */ private[handler] def getResponse(parsedPath: ParsedPath): String = { - JsonConverter.writeToString(toLibraries(parsedPath)) + AdminJsonConverter.writeToString(toLibraries(parsedPath)) } /** package protected for testing */ diff --git a/server/src/main/scala/com/twitter/server/handler/TunableHandler.scala b/server/src/main/scala/com/twitter/server/handler/TunableHandler.scala index 03531d31..22d12baa 100644 --- a/server/src/main/scala/com/twitter/server/handler/TunableHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/TunableHandler.scala @@ -1,14 +1,14 @@ package com.twitter.server.handler import com.twitter.finagle.Service -import com.twitter.finagle.http.{Fields, MediaType, Status, Method, Response, Request} +import com.twitter.finagle.http.{Fields, MediaType, Method, Request, Response, Status} import com.twitter.finagle.tunable.StandardTunableMap import com.twitter.io.Buf +import com.twitter.server.util.AdminJsonConverter import com.twitter.server.util.HttpUtils._ -import com.twitter.server.util.JsonConverter -import com.twitter.util.{Throw, Return, Future} import com.twitter.util.logging.Logger import com.twitter.util.tunable.{JsonTunableMapper, TunableMap} +import com.twitter.util.{Future, Return, Throw} import scala.collection.mutable /** @@ -105,7 +105,7 @@ class TunableHandler private[handler] (registeredIdsFn: () => Map[String, Tunabl respond(Status.NotFound, s"TunableMap not found for id: $id") case Some(map) => val view = toTunableMapView(id, map) - respond(Status.Ok, JsonConverter.writeToString(view)) + respond(Status.Ok, AdminJsonConverter.writeToString(view)) } } @@ -117,7 +117,7 @@ class TunableHandler private[handler] (registeredIdsFn: () => Map[String, Tunabl .map { case (id, map) => toTunableMapView(id, map) } - respond(Status.Ok, JsonConverter.writeToString(view)) + respond(Status.Ok, AdminJsonConverter.writeToString(view)) } private[this] def toTunableMapView(id: String, tunableMap: TunableMap): TunableMapView = { diff --git a/server/src/main/scala/com/twitter/server/util/Json.scala b/server/src/main/scala/com/twitter/server/util/Json.scala index 16bf4b91..6030b97b 100644 --- a/server/src/main/scala/com/twitter/server/util/Json.scala +++ b/server/src/main/scala/com/twitter/server/util/Json.scala @@ -1,20 +1,45 @@ package com.twitter.server.util -private[server] object Json { - import com.fasterxml.jackson.core.`type`.TypeReference - import com.fasterxml.jackson.databind.{ObjectMapper, JsonNode} - import com.fasterxml.jackson.module.scala.DefaultScalaModule - import java.lang.reflect.{Type, ParameterizedType} +import com.fasterxml.jackson.core.JsonFactory +import com.fasterxml.jackson.core.`type`.TypeReference +import com.fasterxml.jackson.core.util.{DefaultIndenter, DefaultPrettyPrinter} +import com.fasterxml.jackson.databind.{ + MappingJsonFactory, + ObjectMapper, + ObjectWriter, + PropertyNamingStrategy +} +import com.fasterxml.jackson.module.scala.DefaultScalaModule +import com.twitter.finagle.http.{Response, Status, Version} +import com.twitter.io.Buf +import java.lang.reflect.{ParameterizedType, Type} + +sealed trait JsonConverterBase { + protected[this] def mapper: ObjectMapper + + protected final val factory = { + val factory = new MappingJsonFactory() + factory.disable(JsonFactory.Feature.USE_THREAD_LOCAL_FOR_BUFFER_RECYCLING) + } - val mapper: ObjectMapper = new ObjectMapper().registerModule(DefaultScalaModule) + private final val writer: ObjectWriter = { + val printer = new DefaultPrettyPrinter + printer.indentArraysWith(DefaultIndenter.SYSTEM_LINEFEED_INSTANCE) - def serialize(o: AnyRef): String = mapper.writeValueAsString(o) + mapper.writer(printer) + } - def deserialize[T: Manifest](value: String): T = - mapper.readValue(value, typeReference[T]) + final def apply(obj: Any): Response = { + val msg = writer.writeValueAsString(obj) + val response = Response(Version.Http11, Status.Ok) + response.content = Buf.Utf8(msg) + response + } - def deserialize[T: Manifest](node: JsonNode): T = - mapper.readValue(node.traverse, typeReference[T]) + final def writeToString(obj: Any): String = writer.writeValueAsString(obj) + + final def parse[T: Manifest](json: String): T = + mapper.readValue(json, typeReference(manifest[T])) private def typeReference[T: Manifest] = new TypeReference[T] { override def getType = typeFromManifest(manifest[T]) @@ -29,3 +54,28 @@ private[server] object Json { def getOwnerType = null } } + +/** + * default Java property naming strategy is used, which leaves field names as is, + * and removes set/get/is prefix from methods (as well as lower-cases initial + * sequence of capitalized characters). + * + * @note Use [[AdminJsonConverter]] for twitter-server endpoints + */ +object JsonConverter extends JsonConverterBase { + protected[this] def mapper: ObjectMapper = + new ObjectMapper(factory) + .registerModule(DefaultScalaModule) +} + +/** + * Twitter Server endpoints should use this converter for the customized ser/deserializers and + * consistent naming strategy. + */ +object AdminJsonConverter extends JsonConverterBase { + protected[this] def mapper: ObjectMapper = + new ObjectMapper(factory) + .registerModule(DefaultScalaModule) + .registerModule(MetricSchemaJsonModule) + .setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE) +} diff --git a/server/src/main/scala/com/twitter/server/util/JsonConverter.scala b/server/src/main/scala/com/twitter/server/util/JsonConverter.scala deleted file mode 100644 index d53b26bb..00000000 --- a/server/src/main/scala/com/twitter/server/util/JsonConverter.scala +++ /dev/null @@ -1,33 +0,0 @@ -package com.twitter.server.util - -import com.fasterxml.jackson.core.JsonFactory -import com.fasterxml.jackson.core.util.{DefaultIndenter, DefaultPrettyPrinter} -import com.fasterxml.jackson.databind.{MappingJsonFactory, ObjectMapper} -import com.fasterxml.jackson.module.scala.DefaultScalaModule -import com.twitter.finagle.http.{Response, Status, Version} -import com.twitter.io.Buf - -object JsonConverter { - private[this] val writer = { - val factory = new MappingJsonFactory() - factory.disable(JsonFactory.Feature.USE_THREAD_LOCAL_FOR_BUFFER_RECYCLING) - val mapper = new ObjectMapper(factory) - .registerModule(DefaultScalaModule) - .registerModule(MetricSchemaJsonModule) - val printer = new DefaultPrettyPrinter - printer.indentArraysWith(DefaultIndenter.SYSTEM_LINEFEED_INSTANCE) - - mapper.writer(printer) - } - - def apply(obj: Any): Response = { - val msg = writer.writeValueAsString(obj) - val response = Response(Version.Http11, Status.Ok) - response.content = Buf.Utf8(msg) - response - } - - def writeToString(obj: Any): String = { - writer.writeValueAsString(obj) - } -} diff --git a/server/src/main/scala/com/twitter/server/util/MetricSchemaJsonModule.scala b/server/src/main/scala/com/twitter/server/util/MetricSchemaJsonModule.scala index fb474d25..602e98e2 100644 --- a/server/src/main/scala/com/twitter/server/util/MetricSchemaJsonModule.scala +++ b/server/src/main/scala/com/twitter/server/util/MetricSchemaJsonModule.scala @@ -1,9 +1,11 @@ package com.twitter.server.util import com.fasterxml.jackson.databind.module.SimpleModule -import com.twitter.server.util.exp.ExpressionSchemaSerializer +import com.twitter.finagle.stats.exp.Operator +import com.twitter.server.util.exp.ExpressionJson.{ExpressionSchemaSerializer, OperatorDeserializer} object MetricSchemaJsonModule extends SimpleModule { addSerializer(SchemaSerializer) addSerializer(ExpressionSchemaSerializer) + addDeserializer(classOf[Operator], OperatorDeserializer) } diff --git a/server/src/main/scala/com/twitter/server/util/exp/ExpressionJson.scala b/server/src/main/scala/com/twitter/server/util/exp/ExpressionJson.scala new file mode 100644 index 00000000..9b6b3b76 --- /dev/null +++ b/server/src/main/scala/com/twitter/server/util/exp/ExpressionJson.scala @@ -0,0 +1,90 @@ +package com.twitter.server.util.exp + +import com.fasterxml.jackson.core.{JsonGenerator, JsonParser} +import com.fasterxml.jackson.databind.{DeserializationContext, SerializerProvider} +import com.fasterxml.jackson.databind.deser.std.StdDeserializer +import com.fasterxml.jackson.databind.ser.std.StdSerializer +import com.twitter.finagle.stats.exp._ +import com.twitter.finagle.stats.metadataScopeSeparator + +/** + * A set of serializers, deserializers used to serve admin/metrics/expressions endpoint + */ +object ExpressionJson { + + /** + * Serialize the ExpressionSchema to JSON + */ + object ExpressionSchemaSerializer + extends StdSerializer[ExpressionSchema](classOf[ExpressionSchema]) { + + def serialize( + expressionSchema: ExpressionSchema, + gen: JsonGenerator, + provider: SerializerProvider + ): Unit = { + gen.writeStartObject() + gen.writeStringField("name", expressionSchema.name) + + gen.writeObjectFieldStart("labels") + gen.writeStringField( + "process_path", + expressionSchema.labels.processPath.getOrElse("Unspecified")) + gen.writeStringField( + "service_name", + expressionSchema.labels.serviceName.getOrElse("Unspecified")) + gen.writeStringField("role", expressionSchema.labels.role.toString) + gen.writeEndObject() + + gen.writeObjectFieldStart("expression") + writeExpression(expressionSchema.expr, gen) + gen.writeEndObject() + + provider.defaultSerializeField("bounds", expressionSchema.bounds, gen) + + gen.writeStringField("description", expressionSchema.description) + gen.writeStringField("unit", expressionSchema.unit.toString) + gen.writeEndObject() + } + + // Temporary, this is the most customized ser/deserialization in expression endpoint. + // Revisit if we need the serializer for ExpressionSchema or Expression. + def writeExpression( + expr: Expression, + gen: JsonGenerator, + name: String = "metric" + ): Unit = { + expr match { + case MetricExpression(schema) => + gen.writeStringField(name, schema.metricBuilder.name.mkString(metadataScopeSeparator())) + case FunctionExpression(funcName, exprs) => + gen.writeStringField("operator", funcName) + gen.writeObjectFieldStart("metrics") + exprs.zipWithIndex.map { + case (expr, index) => writeExpression(expr, gen, name + "-" + index.toString) + } + gen.writeEndObject() + case ConstantExpression(repr, _) => + gen.writeStringField("constant", repr) + } + } + } + + /** + * Deserialize bounds/operator to the case object. + * Serialization is handled by Json annotations. + */ + object OperatorDeserializer extends StdDeserializer[Operator](classOf[Operator]) { + override def deserialize( + parser: JsonParser, + ctx: DeserializationContext + ): Operator = { + parser.readValueAs(classOf[String]) match { + case ">" => GreaterThan + case "<" => LessThan + case other => + throw ctx.instantiationException(classOf[Operator], s"Unknown operator: $other") + } + } + } +} diff --git a/server/src/main/scala/com/twitter/server/util/exp/ExpressionSchemaSerializer.scala b/server/src/main/scala/com/twitter/server/util/exp/ExpressionSchemaSerializer.scala deleted file mode 100644 index 968cde36..00000000 --- a/server/src/main/scala/com/twitter/server/util/exp/ExpressionSchemaSerializer.scala +++ /dev/null @@ -1,65 +0,0 @@ -package com.twitter.server.util.exp - -import com.fasterxml.jackson.core.JsonGenerator -import com.fasterxml.jackson.databind.SerializerProvider -import com.fasterxml.jackson.databind.ser.std.StdSerializer -import com.twitter.finagle.stats.exp.{ - ConstantExpression, - Expression, - ExpressionSchema, - FunctionExpression, - MetricExpression -} -import com.twitter.finagle.stats.metadataScopeSeparator - -object ExpressionSchemaSerializer - extends StdSerializer[ExpressionSchema](classOf[ExpressionSchema]) { - - def serialize( - expressionSchema: ExpressionSchema, - gen: JsonGenerator, - provider: SerializerProvider - ): Unit = { - gen.writeStartObject() - gen.writeStringField("name", expressionSchema.name) - - gen.writeObjectFieldStart("labels") - gen.writeStringField( - "process_path", - expressionSchema.labels.processPath.getOrElse("Unspecified")) - gen.writeStringField( - "service_name", - expressionSchema.labels.serviceName.getOrElse("Unspecified")) - gen.writeStringField("role", expressionSchema.labels.role.toString) - gen.writeEndObject() - - gen.writeObjectFieldStart("expression") - writeExpression(expressionSchema.expr, gen) - gen.writeEndObject() - - gen.writeStringField("bounds", expressionSchema.bounds.toString) - gen.writeStringField("description", expressionSchema.description) - gen.writeStringField("unit", expressionSchema.unit.toString) - gen.writeEndObject() - } - - def writeExpression( - expr: Expression, - gen: JsonGenerator, - name: String = "metric" - ): Unit = { - expr match { - case MetricExpression(schema) => - gen.writeStringField(name, schema.metricBuilder.name.mkString(metadataScopeSeparator())) - case FunctionExpression(funcName, exprs) => - gen.writeStringField("operator", funcName) - gen.writeObjectFieldStart("metrics") - exprs.zipWithIndex.map { - case (expr, index) => writeExpression(expr, gen, name + "-" + index.toString) - } - gen.writeEndObject() - case ConstantExpression(repr, _) => - gen.writeStringField("constant", repr) - } - } -} diff --git a/server/src/main/scala/com/twitter/server/view/BalancersJsonView.scala b/server/src/main/scala/com/twitter/server/view/BalancersJsonView.scala index cc3dcbf3..0b27afcc 100644 --- a/server/src/main/scala/com/twitter/server/view/BalancersJsonView.scala +++ b/server/src/main/scala/com/twitter/server/view/BalancersJsonView.scala @@ -1,7 +1,7 @@ package com.twitter.server.view import com.twitter.finagle.loadbalancer.Metadata -import com.twitter.server.util.JsonConverter +import com.twitter.server.util.AdminJsonConverter private[server] class BalancersJsonView(balancers: Seq[Metadata]) extends View { @@ -24,7 +24,7 @@ private[server] class BalancersJsonView(balancers: Seq[Metadata]) extends View { def render: String = { val clients = balancers.map(renderBalancer) val asMap: Map[String, Object] = Map("clients" -> clients) - JsonConverter.writeToString(asMap) + AdminJsonConverter.writeToString(asMap) } } diff --git a/server/src/test/scala/com/twitter/server/HistogramQueryHandlerTest.scala b/server/src/test/scala/com/twitter/server/HistogramQueryHandlerTest.scala index b8531894..5227bc9b 100644 --- a/server/src/test/scala/com/twitter/server/HistogramQueryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/HistogramQueryHandlerTest.scala @@ -1,6 +1,6 @@ package com.twitter.server -import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.{ObjectMapper, PropertyNamingStrategy} import com.fasterxml.jackson.module.scala.DefaultScalaModule import com.fasterxml.jackson.module.scala.ScalaObjectMapper import com.twitter.conversions.DurationOps._ @@ -15,6 +15,7 @@ class HistogramQueryHandlerTest extends FunSuite { private[this] val mapper = new ObjectMapper with ScalaObjectMapper { registerModule(DefaultScalaModule) } + mapper.setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE) test("histograms.json works with no stats") { val sr = new InMemoryStatsReceiver @@ -44,8 +45,8 @@ class HistogramQueryHandlerTest extends FunSuite { } assert(result.contains("my/cool/stat")) - assert(result.contains(raw""""lowerLimit" : 5,""")) - assert(result.contains(raw""""upperLimit" : 6,""")) + assert(result.contains(raw""""lower_limit" : 5,""")) + assert(result.contains(raw""""upper_limit" : 6,""")) assert(result.contains(raw""""count" : 1""")) } @@ -74,8 +75,8 @@ class HistogramQueryHandlerTest extends FunSuite { assert(upperLimit == 1) assert(percentage == 1.0) } - assert(result.contains(raw""""lowerLimit" : 0,""")) - assert(result.contains(raw""""upperLimit" : 1,""")) + assert(result.contains(raw""""lower_limit" : 0,""")) + assert(result.contains(raw""""upper_limit" : 1,""")) assert(result.contains(raw""""percentage" : 1.0""")) } } diff --git a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala index b2df4afc..d5a62c9b 100644 --- a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala @@ -3,7 +3,7 @@ package com.twitter.server.handler import com.twitter.finagle.http.Request import com.twitter.finagle.stats._ import com.twitter.finagle.stats.exp.ExpressionSchema -import com.twitter.server.util.MetricSchemaSource +import com.twitter.server.util.{JsonUtils, MetricSchemaSource} import com.twitter.util.Await import org.scalatest.FunSuite @@ -314,16 +314,6 @@ class MetricMetadataQueryHandlerTest extends FunSuite { val testNameStart = "MetricTypeQueryHandler generates reasonable json for " - // NOTE: these tests assume a specific iteration order over the registries - // and HashMaps which IS NOT a guarantee. should these tests begin to fail - // due to that, we will need a more robust approach to validation. - private[this] def assertJsonResponse(actual: String, expected: String) = { - assert(stripWhitespace(actual) == stripWhitespace(expected)) - } - - private[this] def stripWhitespace(string: String): String = - string.filter { case c => !c.isWhitespace } - def testCase(latched: Boolean, request: Request): Unit = { if (request == typeRequestNoArg) { testCase(latched, request, testNameStart + "full set of metrics", responseToNoArg) @@ -357,7 +347,7 @@ class MetricMetadataQueryHandlerTest extends FunSuite { | "separator_char" : "/", """.stripMargin test(testName + " when using latched counters") { - assertJsonResponse( + JsonUtils.assertJsonResponse( responseStart + responseMetrics, Await.result(latchedHandler(request)).contentString) } @@ -371,7 +361,7 @@ class MetricMetadataQueryHandlerTest extends FunSuite { | "separator_char" : "/", """.stripMargin test(testName + " when using unlatched counters") { - assertJsonResponse( + JsonUtils.assertJsonResponse( responseStart + responseMetrics, Await.result(unlatchedHandler(request)).contentString) } diff --git a/server/src/test/scala/com/twitter/server/handler/exp/ExpressionSerdeTest.scala b/server/src/test/scala/com/twitter/server/handler/exp/ExpressionSerdeTest.scala new file mode 100644 index 00000000..67fcc5e2 --- /dev/null +++ b/server/src/test/scala/com/twitter/server/handler/exp/ExpressionSerdeTest.scala @@ -0,0 +1,83 @@ +package com.twitter.server.handler.exp + +import com.twitter.finagle.stats.exp.{ + AcceptableRange, + Bounds, + GreaterThan, + LessThan, + MonotoneThresholds, + Unbounded +} +import com.twitter.server.util.{JsonUtils, AdminJsonConverter} +import org.scalatest.FunSuite + +class ExpressionSerdeTest extends FunSuite { + trait BoundsCtx { + val unbounded = Unbounded.get + val monotoneWithUnset = MonotoneThresholds(GreaterThan, 10, 20) + val monotone = MonotoneThresholds(LessThan, 10, 20, Some(0), Some(100)) + val range = AcceptableRange(0, 100) + + val unboundedJson = + """ + |{ + | "kind" : "unbounded" + |}""".stripMargin + + val monotoneWithUnsetJson = + """ + |{ + | "kind" : "monotone", + | "operator" : ">", + | "bad_threshold" : 10.0, + | "good_threshold" : 20.0, + | "lower_bound_inclusive" : null, + | "upper_bound_exclusive" : null + |}""".stripMargin + + val monotoneJson = + """ + |{ + | "kind" : "monotone", + | "operator" : "<", + | "bad_threshold" : 10.0, + | "good_threshold" : 20.0, + | "lower_bound_inclusive" : 0.0, + | "upper_bound_exclusive" : 100.0 + |}""".stripMargin + + val rangeJson = + """ + |{ + | "kind" : "range", + | "lower_bound_inclusive" : 0.0, + | "upper_bound_exclusive" : 100.0 + |}""".stripMargin + } + + test("serialize Bounds to Json") { + new BoundsCtx { + JsonUtils.assertJsonResponse(AdminJsonConverter.writeToString(unbounded), unboundedJson) + JsonUtils.assertJsonResponse( + AdminJsonConverter.writeToString(monotoneWithUnset), + monotoneWithUnsetJson) + JsonUtils.assertJsonResponse(AdminJsonConverter.writeToString(monotone), monotoneJson) + JsonUtils.assertJsonResponse(AdminJsonConverter.writeToString(range), rangeJson) + } + } + + test("deserialize Bounds to object") { + new BoundsCtx { + val unboundedDeserialized = AdminJsonConverter.parse[Bounds](unboundedJson) + val monotoneWithUnsetDeserialized = + AdminJsonConverter.parse[Bounds](monotoneWithUnsetJson) + val monotoneDeserialized = AdminJsonConverter.parse[Bounds](monotoneJson) + val rangeDeserialized = AdminJsonConverter.parse[Bounds](rangeJson) + + assert(unboundedDeserialized == unbounded) + assert(monotoneWithUnsetDeserialized == monotoneWithUnset) + assert(monotoneDeserialized == monotone) + assert(rangeDeserialized == range) + } + } +} diff --git a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala index 6695bb27..b077dc31 100644 --- a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala @@ -1,7 +1,8 @@ package com.twitter.server.handler.exp +import com.twitter.conversions.DurationOps._ import com.twitter.finagle.http.Request -import com.twitter.finagle.stats.exp.{Expression, ExpressionSchema} +import com.twitter.finagle.stats.exp.{Expression, ExpressionSchema, GreaterThan, MonotoneThresholds} import com.twitter.finagle.stats.{ CounterSchema, HistogramSchema, @@ -11,7 +12,7 @@ import com.twitter.finagle.stats.{ SchemaRegistry } import com.twitter.server.handler.MetricExpressionHandler -import com.twitter.server.util.MetricSchemaSource +import com.twitter.server.util.{JsonUtils, MetricSchemaSource} import com.twitter.util.Await import org.scalatest.FunSuite @@ -24,11 +25,15 @@ class MetricExpressionHandlerTest extends FunSuite { CounterSchema(new MetricBuilder(name = Seq("failures"), statsReceiver = sr)) val latencyMb = HistogramSchema(new MetricBuilder(name = Seq("latency"), statsReceiver = sr)) - val successRateExpression = ExpressionSchema( - "success_rate", - Expression(successMb).divide(Expression(successMb).plus(Expression(failuresMb)))) + val successRateExpression = + ExpressionSchema( + "success_rate", + Expression(successMb).divide(Expression(successMb).plus(Expression(failuresMb)))) + .withBounds(MonotoneThresholds(GreaterThan, 99.5, 99.97)) + val throughputExpression = ExpressionSchema("throughput", Expression(successMb).plus(Expression(failuresMb))) + val latencyExpression = ExpressionSchema("latency", Expression(latencyMb)) val expressionSchemaMap: Map[String, ExpressionSchema] = Map( @@ -48,18 +53,13 @@ class MetricExpressionHandlerTest extends FunSuite { val testRequest = Request("http://$HOST:$PORT/admin/metric/expressions.json") - private[this] def assertJsonResponse(actual: String, expected: String) = { - assert(stripWhitespace(actual) == stripWhitespace(expected)) - } - - private[this] def stripWhitespace(string: String): String = - string.filter { case c => !c.isWhitespace } - test("Get the all expressions") { val responseString = """ |{ - | "@version" : 0.1, + | "@version" : 0.2, + | "counters_latched" : false, + | "separator_char" : "/", | "expressions" : [ | { | "name" : "success_rate", @@ -79,7 +79,14 @@ class MetricExpressionHandlerTest extends FunSuite { | } | } | }, - | "bounds" : "Unbounded", + | "bounds" : { + | "kind" : "monotone", + | "operator" : ">", + | "bad_threshold" : 99.5, + | "good_threshold" : 99.97, + | "lower_bound_inclusive" : null, + | "upper_bound_exclusive" : null + | }, | "description" : "Unspecified", | "unit" : "Unspecified" | }, @@ -97,7 +104,9 @@ class MetricExpressionHandlerTest extends FunSuite { | "metric-1" : "failures" | } | }, - | "bounds" : "Unbounded", + | "bounds" : { + | "kind" : "unbounded" + | }, | "description" : "Unspecified", | "unit" : "Unspecified" | }, @@ -111,13 +120,17 @@ class MetricExpressionHandlerTest extends FunSuite { | "expression" : { | "metric" : "latency" | }, - | "bounds" : "Unbounded", + | "bounds" : { + | "kind" : "unbounded" + | }, | "description" : "Unspecified", | "unit" : "Unspecified" | } | ] |}""".stripMargin - assertJsonResponse(responseString, Await.result(expressionHandler(testRequest)).contentString) + JsonUtils.assertJsonResponse( + Await.result(expressionHandler(testRequest), 5.seconds).contentString, + responseString) } } diff --git a/server/src/test/scala/com/twitter/server/util/JsonUtils.scala b/server/src/test/scala/com/twitter/server/util/JsonUtils.scala new file mode 100644 index 00000000..5562e38b --- /dev/null +++ b/server/src/test/scala/com/twitter/server/util/JsonUtils.scala @@ -0,0 +1,17 @@ +package com.twitter.server.util + +import org.scalatest +import org.scalatest.Assertions + +private[server] object JsonUtils { + + // NOTE: these tests assume a specific iteration order over the registries + // and HashMaps which IS NOT a guarantee. should these tests begin to fail + // due to that, we will need a more robust approach to validation. + def assertJsonResponse(actual: String, expected: String): scalatest.Assertion = { + Assertions.assert(stripWhitespace(actual) == stripWhitespace(expected)) + } + + private[this] def stripWhitespace(string: String): String = + string.filter { case c => !c.isWhitespace } +} diff --git a/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala b/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala index 2e2bc192..7b4496e5 100644 --- a/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala +++ b/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala @@ -58,7 +58,7 @@ class MetricSchemaJsonModuleTest extends FunSuite { } test("CounterGauge serializes with kind counter and the correct set of fields") { - val serializedString = JsonConverter.writeToString(counterSchema) + val serializedString = AdminJsonConverter.writeToString(counterSchema) val jsonMap = jsonStrToMap(serializedString) assert(jsonMap.keys == topLevelFieldNameSet) assert( @@ -70,7 +70,7 @@ class MetricSchemaJsonModuleTest extends FunSuite { } test("GaugeSchema serializes with kind gauge and the correct set of fields") { - val serializedString = JsonConverter.writeToString(gaugeSchema) + val serializedString = AdminJsonConverter.writeToString(gaugeSchema) val jsonMap = jsonStrToMap(serializedString) assert(jsonMap.keys == topLevelFieldNameSet) assert( @@ -83,7 +83,7 @@ class MetricSchemaJsonModuleTest extends FunSuite { test( "HistogramSchema serializes with kind histogram and the correct set of fields (includeing buckets)") { - val serializedString = JsonConverter.writeToString(histogramSchema) + val serializedString = AdminJsonConverter.writeToString(histogramSchema) val jsonMap = jsonStrToMap(serializedString) assert(jsonMap.keys == topLevelFieldNameSet ++ Seq("buckets")) assert( From d1cbf38334bad2d522689153a675de164c91a231 Mon Sep 17 00:00:00 2001 From: Matt Dannenberg Date: Fri, 26 Mar 2021 17:58:50 +0000 Subject: [PATCH 236/409] util-stats: document metric_metadata.json endpoint Problem The Metric Metadata Endpoint (/admin/metric_metadata.json) had no documentation, making it tough for some users to decipher the meaning of some fields. Solution Add a section to the Admin.html docs about the Metric Metadata Endpoint. JIRA Issues: CSL-8320 Differential Revision: https://phabricator.twitter.biz/D639225 --- doc/src/sphinx/Admin.rst | 59 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/doc/src/sphinx/Admin.rst b/doc/src/sphinx/Admin.rst index 536eaea7..51f1a2b9 100644 --- a/doc/src/sphinx/Admin.rst +++ b/doc/src/sphinx/Admin.rst @@ -379,6 +379,65 @@ stats to your `statsReceiver` variable. This endpoint is available when you are using the `finagle-stats` library. See the :ref:`metrics ` section for more information. +/admin/metric_metadata.json +~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Export metadata information about metrics in json. By default, this exports metadata for every metric. An optional argument `name` can be added to query for a single metric's metadata. + +:: + + /admin/metric_metadata.json?name=http/requests + +:: + + { + "@version" : 3.0, + "counters_latched" : true, + "separator_char" : "/", + "metrics" : [ + { + "name" : "http/requests", + "relative_name" : [ + "requests" + ], + "kind" : "counter", + "source" : { + "class" : "Unspecified", + "category" : "Server", + "process_path" : "Unspecified" + }, + "description" : "No description provided", + "unit" : "Unspecified", + "verbosity" : "Verbosity(default)", + "key_indicator" : false + } + ] + } + +Top-level/instance-wide Fields: +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:@version: the current version of the response document (format: Major.minor). +:counters_latched: a bool indicating whether or not the service is using latched counters. +:separator_char: the character used to separate path fields in a metric name. +:metrics: an array containing Metadata objects for each metric. + +Metadata Object Fields: +^^^^^^^^^^^^^^^^^^^^^^^ +:name: the fully-qualified name of the metric for this particular Metadata object. +:relative_name: an array containing the field names constituting a suffix which is common across all library users. While the prefix may be configurable, metrics with a shared `relative_name` can be considered alike. +:kind: a string representing the metric type (counter, gauge or histogram). +:source: an object which describes where the metric comes from (see next section for source-level fields). +:description: text description of the metric intended for human consumption. +:unit: the appropriate unit for this metric (ex, milliseconds, megabytes, count). +:verbosity: the metrics logging verbosity level. +:key_indicator: a bool indicating whether or not this metric is part of this services top-level indicators or golden metrics. +:buckets: a map from histogram components (percentiles, max, min, sum, etc) to the suffix which completes this metrics name in metrics.json (only present for histograms). + +Source Object Fields: +^^^^^^^^^^^^^^^^^^^^^ +:class: a string containing the name of the scala/java class that generated this metric (not currently populated). +:category: currently either "Server" or "Client" indicating whether the metric pertains to this server or one of its downstreams, but may be expanded in the future. Should be treated as a string. +:process_path: a string indicating the relevant downstream, when there is one. + Profiling --------- From 48e427ff58b12ee196f19c18532eaf86b34b574d Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Mon, 29 Mar 2021 19:37:35 +0000 Subject: [PATCH 237/409] twitter-server: expression in a query language like format Problem The general "expression" field has not yet been formatted but directly uses JSON to reflect the class structure. Solution Make it a one-line string that uses the common arithmetic function names. Result Serialize expression needs more information (latched/unlatched), we cannot fit it in a Jackson customized Serializer. Added another field in the Expression case class. JIRA Issues: CSL-10618 Differential Revision: https://phabricator.twitter.biz/D638561 --- .../handler/MetricExpressionHandler.scala | 90 ++++++++++++- .../server/util/exp/ExpressionJson.scala | 29 +---- .../exp/MetricExpressionHandlerTest.scala | 120 +++++++++++++----- 3 files changed, 174 insertions(+), 65 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala b/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala index 1f5dacb7..07de6309 100644 --- a/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala @@ -1,22 +1,101 @@ package com.twitter.server.handler import com.twitter.finagle.Service -import com.twitter.finagle.http.{MediaType, Request, Response} -import com.twitter.finagle.stats.metadataScopeSeparator +import com.twitter.finagle.http.{MediaType, Request, Response, Uri} +import com.twitter.finagle.stats.exp.Expression._ +import com.twitter.finagle.stats.exp.{ + ConstantExpression, + Expression, + FunctionExpression, + HistogramExpression, + MetricExpression +} +import com.twitter.finagle.stats.{ + CounterSchema, + HistogramSchema, + MetricSchema, + StatsFormatter, + metadataScopeSeparator +} import com.twitter.io.Buf -import com.twitter.server.handler.MetricExpressionHandler.Version +import com.twitter.server.handler.MetricExpressionHandler.{Version, translateToQuery} import com.twitter.server.util.HttpUtils.newResponse import com.twitter.server.util.{AdminJsonConverter, MetricSchemaSource} import com.twitter.util.Future object MetricExpressionHandler { - private val Version = 0.2 + private val Version = 0.4 + private val statsFormatter = StatsFormatter.default + + /** + * Translate the [[Expression]] object to a single line string which represents generic + * query language. + * @param latched For unlatched counter, we wrap the metric within `rate` + */ + // exposed for testing + private[server] def translateToQuery(expr: Expression, latched: Boolean = false): String = + expr match { + case HistogramExpression(schema, component) => getHisto(schema, component) + case MetricExpression(schema) => getMetric(schema, latched) + case ConstantExpression(repr, _) => repr + case FunctionExpression(funcName, exprs) => + s"$funcName(${exprs.map { expr => translateToQuery(expr, latched) }.mkString(",")})" + } + + // Form a fully formatted name of the histogram with components + // the returned metric is styled the same way as admin/metrics.json + // e.g.request_latency.p9999 or request_latency.min + private def getHisto( + histoSchema: HistogramSchema, + histoComponent: Either[HistogramComponent, Double] + ): String = { + val name = histoSchema.metricBuilder.name.mkString(metadataScopeSeparator()) + val component = histoComponent match { + case Right(percentile) => statsFormatter.labelPercentile(percentile) + case Left(Min) => statsFormatter.labelMin + case Left(Max) => statsFormatter.labelMax + case Left(Avg) => statsFormatter.labelAverage + case Left(Sum) => statsFormatter.labelSum + case Left(Count) => statsFormatter.labelCount + } + statsFormatter.histoName(name, component) + } + + // Form metrics other than histograms, rate() for unlatched counters + private def getMetric( + metricSchema: MetricSchema, + latched: Boolean, + ): String = { + metricSchema match { + case CounterSchema(metricBuilder) if !latched => + s"rate(${metricBuilder.name.mkString(metadataScopeSeparator())})" + case other => other.metricBuilder.name.mkString(metadataScopeSeparator()) + } + } } +/** + * A handler for metric expression queries at admin/metric/expressions.json. + * @queryParam ?latching_style=boolean Set true to let expression respect the latchedness of counters, + * which means it does not wrap the latched counters in `rate()`. + * + * @example http://$HOST:$PORT/admin/metric/expressions.json?latching_style=true + * http://$HOST:$PORT/admin/metric/expressions.json (by default latching_style is false) + */ class MetricExpressionHandler(source: MetricSchemaSource = new MetricSchemaSource) extends Service[Request, Response] { + private[this] lazy val sourceLatched = source.hasLatchedCounters + def apply(request: Request): Future[Response] = { + val keyParam = Uri.fromRequest(request).params.getAll("latching_style") + + val latched = keyParam.exists { value => value == "true" || value == "1" } && sourceLatched + + val expressions = source.expressionList.map { expressionSchema => + expressionSchema.copy(exprQuery = translateToQuery(expressionSchema.expr, latched)) + } + newResponse( contentType = MediaType.JsonUtf8, content = Buf.Utf8( @@ -25,9 +104,10 @@ class MetricExpressionHandler(source: MetricSchemaSource = new MetricSchemaSourc "@version" -> Version, "counters_latched" -> source.hasLatchedCounters, "separator_char" -> metadataScopeSeparator(), - "expressions" -> source.expressionList + "expressions" -> expressions )) ) ) } + } diff --git a/server/src/main/scala/com/twitter/server/util/exp/ExpressionJson.scala b/server/src/main/scala/com/twitter/server/util/exp/ExpressionJson.scala index 9b6b3b76..089a0778 100644 --- a/server/src/main/scala/com/twitter/server/util/exp/ExpressionJson.scala +++ b/server/src/main/scala/com/twitter/server/util/exp/ExpressionJson.scala @@ -1,11 +1,10 @@ package com.twitter.server.util.exp import com.fasterxml.jackson.core.{JsonGenerator, JsonParser} -import com.fasterxml.jackson.databind.{DeserializationContext, SerializerProvider} import com.fasterxml.jackson.databind.deser.std.StdDeserializer import com.fasterxml.jackson.databind.ser.std.StdSerializer +import com.fasterxml.jackson.databind.{DeserializationContext, SerializerProvider} import com.twitter.finagle.stats.exp._ -import com.twitter.finagle.stats.metadataScopeSeparator /** * A set of serializers, deserializers used to serve admin/metrics/expressions endpoint @@ -36,9 +35,7 @@ object ExpressionJson { gen.writeStringField("role", expressionSchema.labels.role.toString) gen.writeEndObject() - gen.writeObjectFieldStart("expression") - writeExpression(expressionSchema.expr, gen) - gen.writeEndObject() + gen.writeStringField("expression", expressionSchema.exprQuery) provider.defaultSerializeField("bounds", expressionSchema.bounds, gen) @@ -46,28 +43,6 @@ object ExpressionJson { gen.writeStringField("unit", expressionSchema.unit.toString) gen.writeEndObject() } - - // Temporary, this is the most customized ser/deserialization in expression endpoint. - // Revisit if we need the serializer for ExpressionSchema or Expression. - def writeExpression( - expr: Expression, - gen: JsonGenerator, - name: String = "metric" - ): Unit = { - expr match { - case MetricExpression(schema) => - gen.writeStringField(name, schema.metricBuilder.name.mkString(metadataScopeSeparator())) - case FunctionExpression(funcName, exprs) => - gen.writeStringField("operator", funcName) - gen.writeObjectFieldStart("metrics") - exprs.zipWithIndex.map { - case (expr, index) => writeExpression(expr, gen, name + "-" + index.toString) - } - gen.writeEndObject() - case ConstantExpression(repr, _) => - gen.writeStringField("constant", repr) - } - } } /** diff --git a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala index b077dc31..6cc213dd 100644 --- a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala @@ -5,6 +5,7 @@ import com.twitter.finagle.http.Request import com.twitter.finagle.stats.exp.{Expression, ExpressionSchema, GreaterThan, MonotoneThresholds} import com.twitter.finagle.stats.{ CounterSchema, + GaugeSchema, HistogramSchema, InMemoryStatsReceiver, MetricBuilder, @@ -12,12 +13,15 @@ import com.twitter.finagle.stats.{ SchemaRegistry } import com.twitter.server.handler.MetricExpressionHandler -import com.twitter.server.util.{JsonUtils, MetricSchemaSource} -import com.twitter.util.Await +import com.twitter.server.util.{AdminJsonConverter, JsonUtils, MetricSchemaSource} +import com.twitter.util.{Await, Awaitable, Duration} import org.scalatest.FunSuite class MetricExpressionHandlerTest extends FunSuite { + private[this] def await[T](awaitable: Awaitable[T], timeout: Duration = 5.second): T = + Await.result(awaitable, timeout) + val sr = new InMemoryStatsReceiver val successMb = CounterSchema(new MetricBuilder(name = Seq("success"), statsReceiver = sr)) @@ -28,18 +32,19 @@ class MetricExpressionHandlerTest extends FunSuite { val successRateExpression = ExpressionSchema( "success_rate", - Expression(successMb).divide(Expression(successMb).plus(Expression(failuresMb)))) - .withBounds(MonotoneThresholds(GreaterThan, 99.5, 99.97)) + Expression(100, sr).multiply( + Expression(successMb).divide(Expression(successMb).plus(Expression(failuresMb)))) + ).withBounds(MonotoneThresholds(GreaterThan, 99.5, 99.97)) val throughputExpression = ExpressionSchema("throughput", Expression(successMb).plus(Expression(failuresMb))) - val latencyExpression = ExpressionSchema("latency", Expression(latencyMb)) + val latencyP99 = ExpressionSchema("latency_p99", Expression(latencyMb, Right(0.99))) val expressionSchemaMap: Map[String, ExpressionSchema] = Map( "success_rate" -> successRateExpression, "throughput" -> throughputExpression, - "latency" -> latencyExpression + "latency" -> latencyP99 ) val expressionRegistry = new SchemaRegistry { @@ -48,16 +53,36 @@ class MetricExpressionHandlerTest extends FunSuite { val expressions: Map[String, ExpressionSchema] = expressionSchemaMap } val expressionSource = new MetricSchemaSource(Seq(expressionRegistry)) - val expressionHandler = new MetricExpressionHandler(expressionSource) + val latchedRegistry = new SchemaRegistry { + def hasLatchedCounters: Boolean = true + def schemas(): Map[String, MetricSchema] = Map.empty + def expressions(): Map[String, ExpressionSchema] = expressionSchemaMap + } + val latchedSource = new MetricSchemaSource(Seq(latchedRegistry)) + val latchedHandler = new MetricExpressionHandler(latchedSource) + val testRequest = Request("http://$HOST:$PORT/admin/metric/expressions.json") + val latchedStyleRequest = Request( + "http://$HOST:$PORT/admin/metric/expressions.json?latching_style=true") + + private def getSucessRateExpression(json: String): String = { + val expressions = + AdminJsonConverter + .parse[Map[String, Any]](json).get("expressions").get.asInstanceOf[List[ + Map[String, String] + ]] + + expressions + .filter(m => m.getOrElse("name", "") == "success_rate").head.getOrElse("expression", "") + } test("Get the all expressions") { - val responseString = + val expectedResponse = """ |{ - | "@version" : 0.2, + | "@version" : 0.4, | "counters_latched" : false, | "separator_char" : "/", | "expressions" : [ @@ -68,17 +93,7 @@ class MetricExpressionHandlerTest extends FunSuite { | "service_name" : "Unspecified", | "role" : "NoRoleSpecified" | }, - | "expression" : { - | "operator" : "divide", - | "metrics" : { - | "metric-0" : "success", - | "operator" : "plus", - | "metrics" : { - | "metric-1-0" : "success", - | "metric-1-1" : "failures" - | } - | } - | }, + | "expression" : "multiply(100.0,divide(rate(success),plus(rate(success),rate(failures))))", | "bounds" : { | "kind" : "monotone", | "operator" : ">", @@ -97,13 +112,7 @@ class MetricExpressionHandlerTest extends FunSuite { | "service_name" : "Unspecified", | "role" : "NoRoleSpecified" | }, - | "expression" : { - | "operator" : "plus", - | "metrics" : { - | "metric-0" : "success", - | "metric-1" : "failures" - | } - | }, + | "expression" : "plus(rate(success),rate(failures))", | "bounds" : { | "kind" : "unbounded" | }, @@ -111,15 +120,13 @@ class MetricExpressionHandlerTest extends FunSuite { | "unit" : "Unspecified" | }, | { - | "name" : "latency", + | "name" : "latency_p99", | "labels" : { | "process_path" : "Unspecified", | "service_name" : "Unspecified", | "role" : "NoRoleSpecified" | }, - | "expression" : { - | "metric" : "latency" - | }, + | "expression" : "latency.p99", | "bounds" : { | "kind" : "unbounded" | }, @@ -130,7 +137,54 @@ class MetricExpressionHandlerTest extends FunSuite { |}""".stripMargin JsonUtils.assertJsonResponse( - Await.result(expressionHandler(testRequest), 5.seconds).contentString, - responseString) + await(expressionHandler(testRequest)).contentString, + expectedResponse) + } + + test("Get the latched expression with ?latched_style=true") { + val responseString = await(latchedHandler(latchedStyleRequest)).contentString + + assert( + getSucessRateExpression( + responseString) == "multiply(100.0,divide(success,plus(success,failures)))") + } + + test("Get the latched expression without latched_style") { + val responseString = await(latchedHandler(testRequest)).contentString + + assert(getSucessRateExpression( + responseString) == "multiply(100.0,divide(rate(success),plus(rate(success),rate(failures))))") + } + + test("translate expressions - counters") { + val latchedResult = + MetricExpressionHandler.translateToQuery(successRateExpression.expr, latched = true) + assert(latchedResult == "multiply(100.0,divide(success,plus(success,failures)))") + + val unlatchedResult = + MetricExpressionHandler.translateToQuery(successRateExpression.expr) + assert( + unlatchedResult == "multiply(100.0,divide(rate(success),plus(rate(success),rate(failures))))") + } + + test("translate histogram expressions - latched does not affect result") { + val latchedResult = MetricExpressionHandler.translateToQuery(latencyP99.expr, latched = true) + val unLatchedResult = MetricExpressionHandler.translateToQuery(latencyP99.expr) + assert(latchedResult == unLatchedResult) + } + + test("translate histogram expressions - components") { + val latencyMinExpr = Expression(latencyMb, Left(Expression.Min)) + val min = MetricExpressionHandler.translateToQuery(latencyMinExpr) + val p99 = MetricExpressionHandler.translateToQuery(latencyP99.expr) + assert(min == "latency.min") + assert(p99 == "latency.p99") + } + + test("translate expressions - gauges") { + val connMb = + GaugeSchema(new MetricBuilder(name = Seq("client", "connections"), statsReceiver = sr)) + val result = MetricExpressionHandler.translateToQuery(Expression(connMb)) + assert(result == "client/connections") } } From 1ec0ee8d2148703e3ae6d8b507c37d26f6d08bcd Mon Sep 17 00:00:00 2001 From: Ian Bennett Date: Mon, 29 Mar 2021 20:58:12 +0000 Subject: [PATCH 238/409] Twitter-oss: Prepare OSS libraries for release 21.3.0 Problem/Solution Let's move the `dodo` CHANGELOG to RST so that it gets updated on each release and prep the release for 21.3.0. JIRA Issues: CSL-10793 Differential Revision: https://phabricator.twitter.biz/D641843 --- CHANGELOG.rst | 3 +++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3ba2a5e2..efc7149b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +21.3.0 +------ + Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ diff --git a/README.md b/README.md index 90042271..d4a12216 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 89e2fa1f..6890e6b9 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.3.0-SNAPSHOT" +val releaseVersion = "21.3.0" val jacksonVersion = "2.11.2" val jacksonLibs = Seq( From 98db8bb0387526636d17c5fb9d7847e10467d302 Mon Sep 17 00:00:00 2001 From: Ian Bennett Date: Mon, 29 Mar 2021 21:38:21 +0000 Subject: [PATCH 239/409] Twitter-oss: update OSS libraries post-release to 21.4.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 21.4.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. JIRA Issues: CSL-10793 Differential Revision: https://phabricator.twitter.biz/D642752 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d4a12216..90042271 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 6890e6b9..64833a82 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.3.0" +val releaseVersion = "21.4.0-SNAPSHOT" val jacksonVersion = "2.11.2" val jacksonLibs = Seq( From be68f1f623288ee5403ac0018249675de0d10a60 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Wed, 31 Mar 2021 21:49:23 +0000 Subject: [PATCH 240/409] {util-stats,twitter-server}: Make it possible to configure counter-ish gauges Problem Some metrics we don't control are essentially counters, tracking the amount of something since the start of an application. However, we don't have a good way of signalling that. Solution Although we considered changing them to be implemented as counters, we were concerned that folks who use latched counters would be surprised that these metrics suddenly started being latched. Although we want to transition in that direction eventually, we chose to do it more cautiously. Instead, we kept the status quo, but added a "counterish_gauge" flag to the metrics metadata, so that if a machine is trying to parse it, they can infer that they may want to take the difference between querying the metrics endpoint. Result The jvm cycles and msec metrics can be more correctly reported! JIRA Issues: CSL-10752 Differential Revision: https://phabricator.twitter.biz/D629892 --- .../server/util/MetricSchemaSerializer.scala | 36 ++++++++++++------- .../util/MetricSchemaJsonModuleTest.scala | 11 ++++++ 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala b/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala index d2892534..f5a860be 100644 --- a/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala +++ b/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala @@ -65,21 +65,31 @@ object SchemaSerializer extends StdSerializer[MetricSchema](classOf[MetricSchema jsonGenerator.writeStringField("verbosity", metricSchema.metricBuilder.verbosity.toString) jsonGenerator.writeBooleanField("key_indicator", metricSchema.metricBuilder.keyIndicator) - if (metricSchema.isInstanceOf[HistogramSchema]) { - jsonGenerator.writeObjectFieldStart("buckets") - jsonGenerator.writeStringField("count", statsFormatter.separator + "count") - jsonGenerator.writeStringField("sum", statsFormatter.separator + "sum") - jsonGenerator.writeStringField( - "average", - statsFormatter.separator + statsFormatter.labelAverage) - jsonGenerator.writeStringField("minimum", statsFormatter.separator + statsFormatter.labelMin) - jsonGenerator.writeStringField("maximum", statsFormatter.separator + statsFormatter.labelMax) - metricSchema.metricBuilder.percentiles.foreach(bucket => + metricSchema match { + case _: GaugeSchema => + if (metricSchema.metricBuilder.isCounterishGauge) { + jsonGenerator.writeBooleanField("counterish_gauge", true) + } + case _: HistogramSchema => + jsonGenerator.writeObjectFieldStart("buckets") + jsonGenerator.writeStringField("count", statsFormatter.separator + "count") + jsonGenerator.writeStringField("sum", statsFormatter.separator + "sum") jsonGenerator.writeStringField( - bucket.toString, - statsFormatter.separator + statsFormatter.labelPercentile(bucket))) + "average", + statsFormatter.separator + statsFormatter.labelAverage) + jsonGenerator.writeStringField( + "minimum", + statsFormatter.separator + statsFormatter.labelMin) + jsonGenerator.writeStringField( + "maximum", + statsFormatter.separator + statsFormatter.labelMax) + metricSchema.metricBuilder.percentiles.foreach(bucket => + jsonGenerator.writeStringField( + bucket.toString, + statsFormatter.separator + statsFormatter.labelPercentile(bucket))) - jsonGenerator.writeEndObject() + jsonGenerator.writeEndObject() + case _ => // nop } jsonGenerator.writeEndObject() } diff --git a/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala b/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala index 7b4496e5..b7ca1189 100644 --- a/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala +++ b/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala @@ -108,4 +108,15 @@ class MetricSchemaJsonModuleTest extends FunSuite { )) } + test("GaugeSchema does not include counterishGauge unless it is true") { + val counterish = gaugeSchema.copy(metricBuilder = gaugeSchema.metricBuilder.withCounterishGauge) + + for (schema <- Seq(gaugeSchema, counterish)) { + val serializedString = AdminJsonConverter.writeToString(schema) + val jsonMap = jsonStrToMap(serializedString) + val expected = if (schema.metricBuilder.isCounterishGauge) Some(true) else None + assert(jsonMap.get("counterish_gauge") == expected) + } + } + } From be8d85224e68a88713c77e9b7db69655124b1ac5 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Thu, 1 Apr 2021 23:06:56 +0000 Subject: [PATCH 241/409] metrics: fix service name setting and correct the constant expression Problem 1.The RoleConfiguredStatsReceiver sets the expression service name either the name is None or Some, LoadedStatsReceiver is sitting at the lower level which reset the service name to None, which makes it possible to be missed when registering in MetricStore. 2.ConstantExpression standalone should not be registered as an expression, hence no need to provide a StatsReceiver. Solution 1. Only set service name when it is not None. 2. Remove StatsReceiver in ConstantExpression. Differential Revision: https://phabricator.twitter.biz/D643761 --- .../com/twitter/server/handler/MetricExpressionHandler.scala | 2 +- .../server/handler/exp/MetricExpressionHandlerTest.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala b/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala index 07de6309..f2cc635a 100644 --- a/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala @@ -37,7 +37,7 @@ object MetricExpressionHandler { expr match { case HistogramExpression(schema, component) => getHisto(schema, component) case MetricExpression(schema) => getMetric(schema, latched) - case ConstantExpression(repr, _) => repr + case ConstantExpression(repr) => repr case FunctionExpression(funcName, exprs) => s"$funcName(${exprs.map { expr => translateToQuery(expr, latched) }.mkString(",")})" } diff --git a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala index 6cc213dd..539910c7 100644 --- a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala @@ -32,7 +32,7 @@ class MetricExpressionHandlerTest extends FunSuite { val successRateExpression = ExpressionSchema( "success_rate", - Expression(100, sr).multiply( + Expression(100).multiply( Expression(successMb).divide(Expression(successMb).plus(Expression(failuresMb)))) ).withBounds(MonotoneThresholds(GreaterThan, 99.5, 99.97)) From 7443d858b2ba9911e9ca07aaecc8dc0ed6dad2ee Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Tue, 6 Apr 2021 23:42:16 +0000 Subject: [PATCH 242/409] metrics: add yufang and mdannenberg as owners Differential Revision: https://phabricator.twitter.biz/D647575 --- PROJECT.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/PROJECT.yml b/PROJECT.yml index 1643d89d..22b76c28 100644 --- a/PROJECT.yml +++ b/PROJECT.yml @@ -2,11 +2,13 @@ owners: - banderson - - hallam - ccoco + - hallam - ibennett - koliver + - mdannenberg - mnakamura - roanta + - yufang watchers: - csl-team:ldap From d76e64d51752a12cc4fdedad57a7b7ad5cb8501e Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Wed, 7 Apr 2021 08:31:37 +0000 Subject: [PATCH 243/409] metrics: Add kernel for memoizing the metric builders Problem Metric metadata including fully scoped names is filled through underlying StatsReceiverProxies. This means the same MetricSchema we passed in Expressions doesn't have any metadata. Solution For metrics we used to build expressions, we hash the MetricBuilder and memorizing in the Metric storage using the object reference hashCode as hashMap key. For registered expressions, replace all MetricBuilders with the fully hydrated ones. Result This is a workaround solution to buy us sometime re-design the StatsReceiver to carry MetricBuilder information so we don't need to back-filling. JIRA Issues: CSL-10845 Differential Revision: https://phabricator.twitter.biz/D645590 --- .../server/handler/MetricExpressionHandler.scala | 2 +- .../handler/MetricMetadataQueryHandlerTest.scala | 8 ++++---- .../handler/exp/MetricExpressionHandlerTest.scala | 10 +++++----- .../server/util/MetricSchemaJsonModuleTest.scala | 6 +++--- .../twitter/server/util/MetricSchemaSourceTest.scala | 6 +++--- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala b/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala index f2cc635a..5c10051c 100644 --- a/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala @@ -24,7 +24,7 @@ import com.twitter.server.util.{AdminJsonConverter, MetricSchemaSource} import com.twitter.util.Future object MetricExpressionHandler { - private val Version = 0.4 + private val Version = 0.6 private val statsFormatter = StatsFormatter.default /** diff --git a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala index d5a62c9b..36093b6f 100644 --- a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala @@ -11,7 +11,7 @@ class MetricMetadataQueryHandlerTest extends FunSuite { val schemaMap: Map[String, MetricSchema] = Map( "my/cool/counter" -> CounterSchema( - new MetricBuilder( + MetricBuilder( keyIndicator = true, description = "Counts how many cools are seen", units = Requests, @@ -24,7 +24,7 @@ class MetricMetadataQueryHandlerTest extends FunSuite { statsReceiver = null )), "your/fine/gauge" -> GaugeSchema( - new MetricBuilder( + MetricBuilder( keyIndicator = false, description = "Measures how fine the downstream system is", units = Percentage, @@ -37,13 +37,13 @@ class MetricMetadataQueryHandlerTest extends FunSuite { statsReceiver = null )), "my/only/histo" -> HistogramSchema( - new MetricBuilder( + MetricBuilder( name = Seq("my", "only", "histo"), percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), statsReceiver = null )), "my/bad/null/counter" -> CounterSchema( - new MetricBuilder( + MetricBuilder( keyIndicator = true, description = "A counter scoped by null get deserialized correctly", units = Requests, diff --git a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala index 539910c7..7d18290a 100644 --- a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala @@ -24,10 +24,10 @@ class MetricExpressionHandlerTest extends FunSuite { val sr = new InMemoryStatsReceiver - val successMb = CounterSchema(new MetricBuilder(name = Seq("success"), statsReceiver = sr)) + val successMb = CounterSchema(MetricBuilder(name = Seq("success"), statsReceiver = sr)) val failuresMb = - CounterSchema(new MetricBuilder(name = Seq("failures"), statsReceiver = sr)) - val latencyMb = HistogramSchema(new MetricBuilder(name = Seq("latency"), statsReceiver = sr)) + CounterSchema(MetricBuilder(name = Seq("failures"), statsReceiver = sr)) + val latencyMb = HistogramSchema(MetricBuilder(name = Seq("latency"), statsReceiver = sr)) val successRateExpression = ExpressionSchema( @@ -82,7 +82,7 @@ class MetricExpressionHandlerTest extends FunSuite { val expectedResponse = """ |{ - | "@version" : 0.4, + | "@version" : 0.6, | "counters_latched" : false, | "separator_char" : "/", | "expressions" : [ @@ -183,7 +183,7 @@ class MetricExpressionHandlerTest extends FunSuite { test("translate expressions - gauges") { val connMb = - GaugeSchema(new MetricBuilder(name = Seq("client", "connections"), statsReceiver = sr)) + GaugeSchema(MetricBuilder(name = Seq("client", "connections"), statsReceiver = sr)) val result = MetricExpressionHandler.translateToQuery(Expression(connMb)) assert(result == "client/connections") } diff --git a/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala b/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala index b7ca1189..82d118ab 100644 --- a/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala +++ b/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala @@ -8,7 +8,7 @@ import org.scalatest.FunSuite class MetricSchemaJsonModuleTest extends FunSuite { private val counterSchema = CounterSchema( - new MetricBuilder( + MetricBuilder( keyIndicator = true, description = "Counts how many cools are seen", units = Requests, @@ -21,7 +21,7 @@ class MetricSchemaJsonModuleTest extends FunSuite { statsReceiver = null )) private val gaugeSchema = GaugeSchema( - new MetricBuilder( + MetricBuilder( keyIndicator = false, description = "Measures how fine the downstream system is", units = Percentage, @@ -34,7 +34,7 @@ class MetricSchemaJsonModuleTest extends FunSuite { statsReceiver = null )) private val histogramSchema = HistogramSchema( - new MetricBuilder( + MetricBuilder( name = Seq("my", "only", "histo"), percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), statsReceiver = null diff --git a/server/src/test/scala/com/twitter/server/util/MetricSchemaSourceTest.scala b/server/src/test/scala/com/twitter/server/util/MetricSchemaSourceTest.scala index 4333ac60..8f001878 100644 --- a/server/src/test/scala/com/twitter/server/util/MetricSchemaSourceTest.scala +++ b/server/src/test/scala/com/twitter/server/util/MetricSchemaSourceTest.scala @@ -8,7 +8,7 @@ class MetricSchemaSourceTest extends FunSuite { private val schemaMap: Map[String, MetricSchema] = Map( "my/cool/counter" -> CounterSchema( - new MetricBuilder( + MetricBuilder( keyIndicator = true, description = "Counts how many cools are seen", units = Requests, @@ -21,7 +21,7 @@ class MetricSchemaSourceTest extends FunSuite { statsReceiver = null )), "your/fine/gauge" -> GaugeSchema( - new MetricBuilder( + MetricBuilder( keyIndicator = false, description = "Measures how fine the downstream system is", units = Percentage, @@ -34,7 +34,7 @@ class MetricSchemaSourceTest extends FunSuite { statsReceiver = null )), "my/only/histo" -> HistogramSchema( - new MetricBuilder( + MetricBuilder( name = Seq("my", "only", "histo"), percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), statsReceiver = null From b46acb3683a0ce2cb73dc47667ae1214f0a4e350 Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Thu, 8 Apr 2021 21:16:54 +0000 Subject: [PATCH 244/409] oss: change github link for CSL projects Problem/Solution Change github links to `blob/release` from `blob/master` JIRA Issues: CSL-10831 Differential Revision: https://phabricator.twitter.biz/D648883 --- CONTRIBUTING.md | 2 +- doc/src/sphinx/Admin.rst | 4 ++-- doc/src/sphinx/Features.rst | 6 +++--- doc/src/sphinx/Migration.rst | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 06cc652b..76bd6bf3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -243,7 +243,7 @@ requests that improve the existing Scaladocs! [6]: https://www.sphinx-doc.org/en/master/usage/installation.html [7]: http://docutils.sourceforge.net/rst.html [8]: https://docs.scala-lang.org/style/scaladoc.html -[9]: https://github.com/twitter/finagle/blob/master/finagle-core/src/test/java/com/twitter/finagle/AddrCompilationTest.java +[9]: https://github.com/twitter/finagle/blob/release/finagle-core/src/test/java/com/twitter/finagle/AddrCompilationTest.java [es]: https://twitter.github.io/effectivescala/ [funsuite]: https://www.scalatest.org/getting_started_with_fun_suite [sbt]: https://www.scala-sbt.org/ diff --git a/doc/src/sphinx/Admin.rst b/doc/src/sphinx/Admin.rst index 51f1a2b9..14ba5f90 100644 --- a/doc/src/sphinx/Admin.rst +++ b/doc/src/sphinx/Admin.rst @@ -243,7 +243,7 @@ registered for `$id`. `PUT` and `DELETE` requests to `/admin/tunables/$id` update the in-memory Tunables for `$id` and should have a JSON body in the same format as a `Tunable` configuration file, detailed in -`JsonTunableMapper `_: +`JsonTunableMapper `_: :: @@ -265,7 +265,7 @@ will *not* cause any existing `Tunables` to be removed. For a `DELETE` request, these `Tunables` will cleared from the `TunableMap`. The `Tunables` are keyed by "id" and "type"; the "value" for each of `Tunables` to delete can be any valid value for this `Tunable`. Because the value of a `Tunable` is the result of a composition of `TunableMaps` -(see `StandardTunableMap `_), deleting an in-memory Tunable will cause the value from the +(see `StandardTunableMap `_), deleting an in-memory Tunable will cause the value from the composition of the other TunableMaps to be used. Metrics diff --git a/doc/src/sphinx/Features.rst b/doc/src/sphinx/Features.rst index 0cf6a3d7..86e7bd67 100644 --- a/doc/src/sphinx/Features.rst +++ b/doc/src/sphinx/Features.rst @@ -14,7 +14,7 @@ Flags .. _flags: The flags implementation, `found in Twitter's util library -`_, +`_, focuses on simplicity and type safety, parsing flags into Scala values. You define your flag like this, in that case the flag type is `String`: @@ -487,12 +487,12 @@ you want to alter the behavior of a trait that is already mixed into `TwitterServer`, you can override methods that you want to have different behavior and then mix it in again. For example, in the `Logging -`_ +`_ trait, you can override loggers to change where you send logs. If you want finer grained control over your server, you can remix traits however you like in the same way that the `TwitterServer -`_ +`_ trait is built. .. _debug metrics: https://twitter.github.io/util/guide/util-stats/basics.html#verbosity-levels diff --git a/doc/src/sphinx/Migration.rst b/doc/src/sphinx/Migration.rst index 5250ea31..39197669 100644 --- a/doc/src/sphinx/Migration.rst +++ b/doc/src/sphinx/Migration.rst @@ -14,7 +14,7 @@ replaces Ostrich’s stats library. in your runtime classpath. By using two -`flags `_, +`flags `_, the admin HTTP endpoint ``/admin/stats.json`` supports an Ostrich compatibility mode: From 7ffed11c5c38187b2d33ce9271c6b8a4f7e4fdf9 Mon Sep 17 00:00:00 2001 From: Matt Dannenberg Date: Tue, 13 Apr 2021 16:24:17 +0000 Subject: [PATCH 245/409] twitter-server: make /admin/metric_metadata.json handle histogram metrics Problem The name argument to /admin/metric_metadata.json did not work properly for histogram metrics when queried by the names they surface under in /admin/metrics.json Solution If the name argument to /admin/metric_metadata.json contains the histogramSeparator character, truncate the metric name and return the full Histogram metadata for that metric. JIRA Issues: OBSERVE-32997 Differential Revision: https://phabricator.twitter.biz/D640725 --- CHANGELOG.rst | 3 + doc/src/sphinx/Admin.rst | 37 +++++- .../handler/MetricMetadataQueryHandler.scala | 25 +++- .../server/util/MetricSchemaSerializer.scala | 12 +- .../MetricMetadataQueryHandlerTest.scala | 115 +++++++++++++++++- 5 files changed, 175 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index efc7149b..c23e8cd5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +* Change Metrics Metadata Endpoint to return a histogram's metadata when queried using the /admin/metrics.json + full (suffixed) histogram name as the value for the name argument. ``PHAB_ID=D640725`` + 21.3.0 ------ diff --git a/doc/src/sphinx/Admin.rst b/doc/src/sphinx/Admin.rst index 14ba5f90..a8a1ee26 100644 --- a/doc/src/sphinx/Admin.rst +++ b/doc/src/sphinx/Admin.rst @@ -381,16 +381,18 @@ See the :ref:`metrics ` section for more information. /admin/metric_metadata.json ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Export metadata information about metrics in json. By default, this exports metadata for every metric. An optional argument `name` can be added to query for a single metric's metadata. +Export metadata information about metrics in json. By default, this exports metadata for every metric. An optional argument `name` can be added to query for a single metric's metadata. Multiple names may be passed in using that argument multiple times. + +NB: for a histogram, it may be looked up with our without its metrics.json suffix (.sum, .p95, .max, etc). :: - /admin/metric_metadata.json?name=http/requests + /admin/metric_metadata.json?name=http/requests&name=strato_rpc_client/latency_ms.p95 :: { - "@version" : 3.0, + "@version" : 3.1, "counters_latched" : true, "separator_char" : "/", "metrics" : [ @@ -409,6 +411,35 @@ Export metadata information about metrics in json. By default, this exports meta "unit" : "Unspecified", "verbosity" : "Verbosity(default)", "key_indicator" : false + }, + { + "name" : "strato_rpc_client/latency_ms", + "relative_name" : [ + "latency_ms" + ], + "kind" : "histogram", + "source" : { + "class" : "Unspecified", + "category" : "Client", + "process_path" : "Unspecified" + }, + "description" : "No description provided", + "unit" : "Unspecified", + "verbosity" : "Verbosity(default)", + "key_indicator" : false, + "buckets" : { + "count" : ".count", + "sum" : ".sum", + "average" : ".avg", + "minimum" : ".min", + "maximum" : ".max", + "0.5" : ".p50", + "0.9" : ".p90", + "0.95" : ".p95", + "0.99" : ".p99", + "0.999" : ".p9990", + "0.9999" : ".p9999" + } } ] } diff --git a/server/src/main/scala/com/twitter/server/handler/MetricMetadataQueryHandler.scala b/server/src/main/scala/com/twitter/server/handler/MetricMetadataQueryHandler.scala index 24d99bde..84204958 100644 --- a/server/src/main/scala/com/twitter/server/handler/MetricMetadataQueryHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/MetricMetadataQueryHandler.scala @@ -2,7 +2,7 @@ package com.twitter.server.handler import com.twitter.finagle.Service import com.twitter.finagle.http.{MediaType, Request, Response, Uri} -import com.twitter.finagle.stats.metadataScopeSeparator +import com.twitter.finagle.stats.{StatsFormatter, metadataScopeSeparator} import com.twitter.io.Buf import com.twitter.server.util.HttpUtils.newResponse import com.twitter.server.util.{AdminJsonConverter, MetricSchemaSource} @@ -84,8 +84,25 @@ import com.twitter.util.Future class MetricMetadataQueryHandler(source: MetricSchemaSource = new MetricSchemaSource) extends Service[Request, Response] { - private[this] def query(keys: Iterable[String]) = - for (k <- keys; e <- source.getSchema(k)) yield e + private[this] val statsFormatter = StatsFormatter.default + + private[this] def query(keys: Iterable[String]) = { + keys.flatMap(k => { + // histogram metric. + if (k.lastIndexOf(statsFormatter.histogramSeparator) > k.lastIndexOf( + metadataScopeSeparator())) { + val trimmedMetricName = + if (k.endsWith("percentile")) { // see StatsFormatter.CommonsStats + val intermediate = k.substring(0, k.lastIndexOf(statsFormatter.histogramSeparator)) + intermediate.substring(0, intermediate.lastIndexOf(statsFormatter.histogramSeparator)) + } else k.substring(0, k.lastIndexOf(statsFormatter.histogramSeparator)) + val metric = source.getSchema(trimmedMetricName) + if (metric.isEmpty) source.getSchema(k) else metric + } else { + source.getSchema(k) + } + }) + } def apply(req: Request): Future[Response] = { val uri = Uri.fromRequest(req) @@ -102,7 +119,7 @@ class MetricMetadataQueryHandler(source: MetricSchemaSource = new MetricSchemaSo content = Buf.Utf8( AdminJsonConverter.writeToString( Map( - "@version" -> 3.0, + "@version" -> 3.1, "counters_latched" -> latched, "separator_char" -> metadataScopeSeparator(), "metrics" -> metrics diff --git a/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala b/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala index f5a860be..14598ed8 100644 --- a/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala +++ b/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala @@ -72,21 +72,21 @@ object SchemaSerializer extends StdSerializer[MetricSchema](classOf[MetricSchema } case _: HistogramSchema => jsonGenerator.writeObjectFieldStart("buckets") - jsonGenerator.writeStringField("count", statsFormatter.separator + "count") - jsonGenerator.writeStringField("sum", statsFormatter.separator + "sum") + jsonGenerator.writeStringField("count", statsFormatter.histogramSeparator + "count") + jsonGenerator.writeStringField("sum", statsFormatter.histogramSeparator + "sum") jsonGenerator.writeStringField( "average", - statsFormatter.separator + statsFormatter.labelAverage) + statsFormatter.histogramSeparator + statsFormatter.labelAverage) jsonGenerator.writeStringField( "minimum", - statsFormatter.separator + statsFormatter.labelMin) + statsFormatter.histogramSeparator + statsFormatter.labelMin) jsonGenerator.writeStringField( "maximum", - statsFormatter.separator + statsFormatter.labelMax) + statsFormatter.histogramSeparator + statsFormatter.labelMax) metricSchema.metricBuilder.percentiles.foreach(bucket => jsonGenerator.writeStringField( bucket.toString, - statsFormatter.separator + statsFormatter.labelPercentile(bucket))) + statsFormatter.histogramSeparator + statsFormatter.labelPercentile(bucket))) jsonGenerator.writeEndObject() case _ => // nop diff --git a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala index 36093b6f..289a2682 100644 --- a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala @@ -4,7 +4,7 @@ import com.twitter.finagle.http.Request import com.twitter.finagle.stats._ import com.twitter.finagle.stats.exp.ExpressionSchema import com.twitter.server.util.{JsonUtils, MetricSchemaSource} -import com.twitter.util.Await +import com.twitter.util.{Await, Duration} import org.scalatest.FunSuite class MetricMetadataQueryHandlerTest extends FunSuite { @@ -98,7 +98,7 @@ class MetricMetadataQueryHandlerTest extends FunSuite { "http://$HOST:$PORT/admin/metric_metadata.json?name=my/only/histo") val typeRequestWithHistoAndNon = Request( - "http://$HOST:$PORT/admin/metric_metadata.json?name=my/cool/counter&name=my/only/histo") + "http://$HOST:$PORT/admin/metric_metadata.json?name=my/cool/counter&name=my/only/histo.p99") val responseToNoArg = """ @@ -342,7 +342,7 @@ class MetricMetadataQueryHandlerTest extends FunSuite { val responseStart = """ | { - | "@version" : 3.0, + | "@version" : 3.1, | "counters_latched" : true, | "separator_char" : "/", """.stripMargin @@ -356,7 +356,7 @@ class MetricMetadataQueryHandlerTest extends FunSuite { val responseStart = """ | { - | "@version" : 3.0, + | "@version" : 3.1, | "counters_latched" : false, | "separator_char" : "/", """.stripMargin @@ -381,4 +381,111 @@ class MetricMetadataQueryHandlerTest extends FunSuite { testCase(false, typeRequestWithHisto), testCase(false, typeRequestWithHistoAndNon) ) + + /** + * Histogram Metric Querying cases. + */ + trait SchemalessRegistry extends SchemaRegistry { + val hasLatchedCounters = false + def expressions(): Map[String, ExpressionSchema] = Map.empty + } + + def histoSuffixTestCase( + testName: String, + metricName: String, + requestParam: String, + metricType: String, + useCommonsStats: Boolean = false + ): Unit = { + test(testName) { + trait registry extends SchemalessRegistry { + def schemas(): Map[String, MetricSchema] = if (metricType == "counter") { + Map( + metricName -> CounterSchema( + MetricBuilder( + name = metricName.split("\\/"), + percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), + statsReceiver = null + ))) + } else { + Map( + metricName -> HistogramSchema( + MetricBuilder( + name = metricName.split("\\/"), + percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), + statsReceiver = null + ))) + } + } + val request = Request("http://$HOST:$PORT/admin/metric_metadata.json?name=" + requestParam) + if (useCommonsStats) { + format.let("commonsstats") { + val handler = + new MetricMetadataQueryHandler(new MetricSchemaSource(Seq(new registry() {}))) + assert( + Await + .result(handler(request), Duration.fromSeconds(10)) + .contentString + .contains("kind\" : \"%s\"".format(metricType))) + } + } else { + val handler = + new MetricMetadataQueryHandler(new MetricSchemaSource(Seq(new registry() {}))) + assert( + Await + .result(handler(request), Duration.fromSeconds(10)).contentString.contains( + "kind\" : \"%s\"".format(metricType))) + } + } + } + + histoSuffixTestCase( + "counter with histo separator in name querability test", + "foo/bar.baz", + "foo/bar.baz", + "counter" + ) + + histoSuffixTestCase( + "counter with histobucket-like name querability test", + "foo/bar.p999", + "foo/bar.p999", + "counter" + ) + + histoSuffixTestCase( + "histogram querability via bucket name test", + "foo", + "foo.p999", + "histogram" + ) + + histoSuffixTestCase( + "histogram querability via raw name test", + "foo", + "foo", + "histogram" + ) + + histoSuffixTestCase( + "counter with less common histo separator in name querability test", + "foo/bar_baz", + "foo/bar_baz", + "counter", + true) + + histoSuffixTestCase( + "histo with less common histo separator in name querability test", + "foo/bar_baz", + "foo/bar_baz", + "histogram", + true) + + histoSuffixTestCase( + "histo with less common histo separator in name percentile querability test", + "foo/bar_baz", + "foo/bar_baz_99_percentile", + "histogram", + true) + } From bcd57c0c114660d6276256ce6437b6ee01262aff Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Mon, 26 Apr 2021 21:05:08 +0000 Subject: [PATCH 246/409] Twitter-oss: Prepare OSS libraries for release 21.4 Problem We want to release the next versions of our Twitter OSS libraries 21.4.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-10889 Differential Revision: https://phabricator.twitter.biz/D658986 --- CHANGELOG.rst | 3 +++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c23e8cd5..53694666 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +21.4.0 +------ + * Change Metrics Metadata Endpoint to return a histogram's metadata when queried using the /admin/metrics.json full (suffixed) histogram name as the value for the name argument. ``PHAB_ID=D640725`` diff --git a/README.md b/README.md index 90042271..d4a12216 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 64833a82..44f47f6c 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.4.0-SNAPSHOT" +val releaseVersion = "21.4.0" val jacksonVersion = "2.11.2" val jacksonLibs = Seq( From af933c0d21b6db13629682bf441ddb607588d526 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Mon, 26 Apr 2021 21:52:22 +0000 Subject: [PATCH 247/409] Twitter-oss: update OSS libraries post-release to 21.5-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 21.5-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. JIRA Issues: CSL-10889 Differential Revision: https://phabricator.twitter.biz/D659035 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d4a12216..90042271 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 44f47f6c..aa87ae6d 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.4.0" +val releaseVersion = "21.5.0-SNAPSHOT" val jacksonVersion = "2.11.2" val jacksonLibs = Seq( From 87aefd8ee2b1daa62a627828f4c2cd93b0cce524 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Wed, 28 Apr 2021 01:04:40 +0000 Subject: [PATCH 248/409] metric-expressions: add the field namespaces for multitenancy support Problem Multitenancy system would like to instrument expressions pre-tenant, so each tenant can get its own monitoring and alerting. Expression APIs currently don't have this supported. Solution Add a field in expressions, multitenancy systems can define expressions in their layers for each tenant. JIRA Issues: CSL-10810 Differential Revision: https://phabricator.twitter.biz/D657317 --- .../twitter/server/handler/MetricExpressionHandler.scala | 2 +- .../com/twitter/server/util/exp/ExpressionJson.scala | 6 ++++++ .../server/handler/exp/MetricExpressionHandlerTest.scala | 8 ++++++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala b/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala index 5c10051c..43464be4 100644 --- a/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala @@ -24,7 +24,7 @@ import com.twitter.server.util.{AdminJsonConverter, MetricSchemaSource} import com.twitter.util.Future object MetricExpressionHandler { - private val Version = 0.6 + private val Version = 0.7 private val statsFormatter = StatsFormatter.default /** diff --git a/server/src/main/scala/com/twitter/server/util/exp/ExpressionJson.scala b/server/src/main/scala/com/twitter/server/util/exp/ExpressionJson.scala index 089a0778..d60c1ae2 100644 --- a/server/src/main/scala/com/twitter/server/util/exp/ExpressionJson.scala +++ b/server/src/main/scala/com/twitter/server/util/exp/ExpressionJson.scala @@ -35,6 +35,12 @@ object ExpressionJson { gen.writeStringField("role", expressionSchema.labels.role.toString) gen.writeEndObject() + if (expressionSchema.namespaces.nonEmpty) { + gen.writeArrayFieldStart("namespaces") + expressionSchema.namespaces.foreach { p => gen.writeString(p) } + gen.writeEndArray() + } + gen.writeStringField("expression", expressionSchema.exprQuery) provider.defaultSerializeField("bounds", expressionSchema.bounds, gen) diff --git a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala index 7d18290a..86cad593 100644 --- a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala @@ -38,8 +38,10 @@ class MetricExpressionHandlerTest extends FunSuite { val throughputExpression = ExpressionSchema("throughput", Expression(successMb).plus(Expression(failuresMb))) + .withNamespaces("path", "to", "tenantName") - val latencyP99 = ExpressionSchema("latency_p99", Expression(latencyMb, Right(0.99))) + val latencyP99 = + ExpressionSchema("latency_p99", Expression(latencyMb, Right(0.99))).withNamespaces("tenantName") val expressionSchemaMap: Map[String, ExpressionSchema] = Map( "success_rate" -> successRateExpression, @@ -82,7 +84,7 @@ class MetricExpressionHandlerTest extends FunSuite { val expectedResponse = """ |{ - | "@version" : 0.6, + | "@version" : 0.7, | "counters_latched" : false, | "separator_char" : "/", | "expressions" : [ @@ -112,6 +114,7 @@ class MetricExpressionHandlerTest extends FunSuite { | "service_name" : "Unspecified", | "role" : "NoRoleSpecified" | }, + | "namespaces" : ["path","to","tenantName"], | "expression" : "plus(rate(success),rate(failures))", | "bounds" : { | "kind" : "unbounded" @@ -126,6 +129,7 @@ class MetricExpressionHandlerTest extends FunSuite { | "service_name" : "Unspecified", | "role" : "NoRoleSpecified" | }, + | "namespaces" : ["tenantName"], | "expression" : "latency.p99", | "bounds" : { | "kind" : "unbounded" From 6d934631ed853d582737d355873bd7e4356d4d9e Mon Sep 17 00:00:00 2001 From: Adam Singer Date: Thu, 29 Apr 2021 15:22:53 +0000 Subject: [PATCH 249/409] [twitter-server] Remove rglobs on slf4j-jdk14 libraries Problem `twitter-server/slf4j-jdk14/src/main/scala/BUILD` accidentally includes sources from `twitter-server/slf4j-jdk14/src/main/scala/com/twitter/server/logging/*.scala` and `twitter-server/slf4j-jdk14/src/main/scala/com/twitter/server/slf4j/jdk14/*.scala`. Solution Remove the rglobs from `twitter-server/slf4j-jdk14/src/main/scala/BUILD` and directly reference `twitter-server/slf4j-jdk14/src/main/scala/com/twitter/server/handler/slf4j/jdk14/*.scala`. JIRA Issues: CSL-10903 Differential Revision: https://phabricator.twitter.biz/D652574 --- slf4j-jdk14/src/main/scala/BUILD | 4 +++- .../com/twitter/server/handler/slf4j/jdk14/BUILD | 16 ---------------- slf4j-jdk14/src/test/scala/BUILD | 4 ---- 3 files changed, 3 insertions(+), 21 deletions(-) delete mode 100644 slf4j-jdk14/src/main/scala/com/twitter/server/handler/slf4j/jdk14/BUILD diff --git a/slf4j-jdk14/src/main/scala/BUILD b/slf4j-jdk14/src/main/scala/BUILD index d3715b30..a10cca07 100644 --- a/slf4j-jdk14/src/main/scala/BUILD +++ b/slf4j-jdk14/src/main/scala/BUILD @@ -1,5 +1,5 @@ scala_library( - sources = ["**/*.scala"], + sources = ["com/twitter/server/handler/slf4j/jdk14/*.scala"], compiler_option_sets = ["fatal_warnings"], provides = scala_artifact( org = "com.twitter", @@ -7,6 +7,7 @@ scala_library( repo = artifactory, ), scope = "forced", # Exposes a service-loaded class. + strict_deps = True, tags = ["logging_impl_check_whitelisted_target"], dependencies = [ "3rdparty/jvm/org/slf4j:slf4j-api", @@ -23,6 +24,7 @@ scala_library( "finagle/finagle-http/src/main/scala", "twitter-server/server/src/main/scala", "twitter-server/slf4j-jdk14/src/main/resources", + "util/util-core/src/main/scala/com/twitter/io", "util/util-logging/src/main/scala", ], exports = [ diff --git a/slf4j-jdk14/src/main/scala/com/twitter/server/handler/slf4j/jdk14/BUILD b/slf4j-jdk14/src/main/scala/com/twitter/server/handler/slf4j/jdk14/BUILD deleted file mode 100644 index a941f6fb..00000000 --- a/slf4j-jdk14/src/main/scala/com/twitter/server/handler/slf4j/jdk14/BUILD +++ /dev/null @@ -1,16 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - provides = scala_artifact( - org = "com.twitter", - name = "twitter-server-slf4j-jdk14-handler", - repo = artifactory, - ), - strict_deps = True, - dependencies = [ - "finagle/finagle-http/src/main/scala", - "twitter-server/server/src/main/scala", - "util/util-core/src/main/scala/com/twitter/io", - "util/util-logging/src/main/scala/com/twitter/logging", - ], -) diff --git a/slf4j-jdk14/src/test/scala/BUILD b/slf4j-jdk14/src/test/scala/BUILD index d1927384..3a585299 100644 --- a/slf4j-jdk14/src/test/scala/BUILD +++ b/slf4j-jdk14/src/test/scala/BUILD @@ -12,10 +12,6 @@ junit_tests( "twitter-server/slf4j-jdk14/src/main/scala", scope = "runtime", ), - scoped( - "twitter-server/slf4j-jdk14/src/main/scala/com/twitter/server/handler/slf4j/jdk14", - scope = "runtime" - ), "finagle/finagle-base-http/src/main", "finagle/finagle-core/src/main", "finagle/finagle-http/src/main/scala", From bbef3149e5cf34f0b7e453d63b0f50ecf196a7bd Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Tue, 4 May 2021 15:42:47 +0000 Subject: [PATCH 250/409] twitter-server: Document counterish gauges Problem / Solution Now that there are metadata docs, we should document counterish gauges. Differential Revision: https://phabricator.twitter.biz/D648366 --- doc/src/sphinx/Admin.rst | 1 + doc/src/sphinx/docutils.conf | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 doc/src/sphinx/docutils.conf diff --git a/doc/src/sphinx/Admin.rst b/doc/src/sphinx/Admin.rst index a8a1ee26..4725d1ae 100644 --- a/doc/src/sphinx/Admin.rst +++ b/doc/src/sphinx/Admin.rst @@ -461,6 +461,7 @@ Metadata Object Fields: :unit: the appropriate unit for this metric (ex, milliseconds, megabytes, count). :verbosity: the metrics logging verbosity level. :key_indicator: a bool indicating whether or not this metric is part of this services top-level indicators or golden metrics. +:counterish_gauge: a bool indicating if a gauge represents a count of something over the lifetime of the instance. Note that it does not follow the same latching rules as standard counters. This key will not be present unless it is true. It can only be present on gauge-type metrics. :buckets: a map from histogram components (percentiles, max, min, sum, etc) to the suffix which completes this metrics name in metrics.json (only present for histograms). Source Object Fields: diff --git a/doc/src/sphinx/docutils.conf b/doc/src/sphinx/docutils.conf new file mode 100644 index 00000000..e3ddf4c0 --- /dev/null +++ b/doc/src/sphinx/docutils.conf @@ -0,0 +1,2 @@ +[html4css1 writer] +field_name_limit = 0 From 6fb730e662b4c0ac91a85e6934bee0b225ba0d82 Mon Sep 17 00:00:00 2001 From: Matt Dannenberg Date: Fri, 7 May 2021 15:07:37 +0000 Subject: [PATCH 251/409] util-stats: document metric/expressions.json endpoint Problem / Solution initial documentation for admin/metric/expressions.json. i think a diagram might help explain the bounds, but we may wanna wait until that stuff is more mature or successfully adopted before doing so. JIRA Issues: CSL-10715 Differential Revision: https://phabricator.twitter.biz/D663147 --- doc/src/sphinx/Admin.rst | 69 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/doc/src/sphinx/Admin.rst b/doc/src/sphinx/Admin.rst index 4725d1ae..e7ab4245 100644 --- a/doc/src/sphinx/Admin.rst +++ b/doc/src/sphinx/Admin.rst @@ -470,6 +470,75 @@ Source Object Fields: :category: currently either "Server" or "Client" indicating whether the metric pertains to this server or one of its downstreams, but may be expanded in the future. Should be treated as a string. :process_path: a string indicating the relevant downstream, when there is one. +/admin/metric/expressions.json +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Export metadata information about metric expressions in json. These metadata represent commonly used metric queries for the service. + +:: + + /admin/metric/expressions.json?name=http/requests&name=strato_rpc_client/latency_ms.p95 + +:: + + { + "@version" : 0.7, + "counters_latched" : true, + "separator_char" : "/", + "expressions" : [ + { + "name" : "success_rate", + "labels" : { + "process_path" : "Unspecified", + "service_name" : "thrift", + "role" : "Client" + }, + "expression" : "multiply(100.0,divide(rate(clnt/thrift/success),plus(rate(clnt/thrift/success),rate(clnt/thrift/failures))))", + "bounds" : { + "kind" : "monotone", + "operator" : ">", + "bad_threshold" : 99.5, + "good_threshold" : 99.97, + "lower_bound_inclusive" : null, + "upper_bound_exclusive" : null + }, + "description" : "The Success Rate", + "unit" : "Percentage" + }, + ] + } + +Top-level/instance-wide Fields: +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:@version: the current version of the response document (format: Major.minor). +:counters_latched: a bool indicating whether or not the service is using latched counters. +:separator_char: the character used to separate path fields in a metric name. +:expressions: an array containing a Metadata object for each metric expression. + +Expression Object Fields: +^^^^^^^^^^^^^^^^^^^^^^^^^ +:name: a string representing the name or significance of the expression. +:labels: an object which describes where the metric comes from (see next section for labels-level fields). +:expresssion: a string representation illustrating how to aggregate metrics into the expression value. +:bounds: an object which describes the bounds for what values resulting from the expression should be considered "healthy" for the service. +:description: text description of the expression, intended for human consumption. +:unit: the appropriate unit for this metric (ex, milliseconds, megabytes, count). + +Labels Object Fields: +^^^^^^^^^^^^^^^^^^^^^ +This object is likely to change content and even become more flexible in the future. It is safest to treat it as a Map[String, String]. +:process_path: a string indicating the relevant downstream, when there is one. +:service_name: a string representing the finagle service name. particularly helpful for services containing multiple internal servers (ex, thrift and http). +:role: either "Server" or "Client" indicating whether the expression pertains to this server or one of its downstreams. Should be treated as a string. + +Bounds Object Fields: +^^^^^^^^^^^^^^^^^^^^^ +:kind: a string indicating what type of bounds these are: "unbounded" (no bounds), "monotone" (only considered unhealthy in one direction), or "range" (there is a range of healthy values, above or below them is unhealthy). +:operator: a string indicating which whether the bounds should be treated as floors or ceilings (ex, ">", "<"). +:bad_threshold: the value that represents the boundary between a bad and an ok value for the expression. +:good_threshold: the value that represents the boundary between an ok and a good value for the expression. +:lower_bound_inclusive: an optional value below the bad_threshold which indicates a boundry below which values are considered invalid or erroneous rather than unhealthy. +:upper_bound_exclusive: an optional value above the good_threshold which indicates a boundry above which values are considered invalid or erroneous rather than unhealthy. + Profiling --------- From 052ddb3f9e4e0d166e805ad18b7c4c2f71684dad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Abecasis?= Date: Tue, 11 May 2021 16:29:43 +0000 Subject: [PATCH 252/409] twitter-server: Update histograms docs and admin page to use snake case Problem The histograms admin endpoint was updated in c90fa00caeba to use snake_case for field names. The same change missed updating documentation for the endpoint and the JavaScript that supports histograms in the admin interface. Solution Updated docs and JS code to use snake_case field names for upper and lower limits of histogram buckets. Closes twitter/twitter-server#68 Signed-off-by: Lily He JIRA Issues: CSL-10945 Differential Revision: https://phabricator.twitter.biz/D667345 --- doc/src/sphinx/Features.rst | 56 +++++++++---------- .../twitter-server/js/histogram-dom.js | 2 +- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/doc/src/sphinx/Features.rst b/doc/src/sphinx/Features.rst index 86e7bd67..5f62fc6e 100644 --- a/doc/src/sphinx/Features.rst +++ b/doc/src/sphinx/Features.rst @@ -307,35 +307,35 @@ There are two endpoints for histograms, `/admin/histograms.json` and `/admin/his { "clnt/p2cslowservertest-server/request_latency_ms" : [ { - "lowerLimit" : 5, - "upperLimit" : 6, + "lower_limit" : 5, + "upper_limit" : 6, "count" : 28 }, { - "lowerLimit" : 6, - "upperLimit" : 7, + "lower_limit" : 6, + "upper_limit" : 7, "count" : 9188 }, { - "lowerLimit" : 7, - "upperLimit" : 8, + "lower_limit" : 7, + "upper_limit" : 8, "count" : 25164 } ], "jvm/gc/eden/pause_msec" : [ { - "lowerLimit" : 8, - "upperLimit" : 9, + "lower_limit" : 8, + "upper_limit" : 9, "count" : 1 }, { - "lowerLimit" : 9, - "upperLimit" : 10, + "lower_limit" : 9, + "upper_limit" : 10, "count" : 1 }, { - "lowerLimit" : 53, - "upperLimit" : 54, + "lower_limit" : 53, + "upper_limit" : 54, "count" : 1 } ] @@ -373,18 +373,18 @@ equal to the bucket's upper limit. { "clnt/p2cslowservertest-server/request_latency_ms" : [ { - "lowerLimit" : 5, - "upperLimit" : 6, + "lower_limit" : 5, + "upper_limit" : 6, "count" : 28 }, { - "lowerLimit" : 6, - "upperLimit" : 7, + "lower_limit" : 6, + "upper_limit" : 7, "count" : 9188 }, { - "lowerLimit" : 7, - "upperLimit" : 8, + "lower_limit" : 7, + "upper_limit" : 8, "count" : 25164 } ] @@ -401,28 +401,28 @@ equal to the bucket's upper limit. { "clnt/p2cslowservertest-server/request_latency_ms" : [ { - "lowerLimit" : 5, - "upperLimit" : 6, + "lower_limit" : 5, + "upper_limit" : 6, "percentage" : 6.444885E-5 }, { - "lowerLimit" : 6, - "upperLimit" : 7, + "lower_limit" : 6, + "upper_limit" : 7, "percentage" : 0.03286891 }, { - "lowerLimit" : 7, - "upperLimit" : 8, + "lower_limit" : 7, + "upper_limit" : 8, "percentage" : 0.15292539 }, { - "lowerLimit" : 8, - "upperLimit" : 9, + "lower_limit" : 8, + "upper_limit" : 9, "percentage" : 0.26998794 }, { - "lowerLimit" : 9, - "upperLimit" : 10, + "lower_limit" : 9, + "upper_limit" : 10, "percentage" : 0.41753477 } ] diff --git a/server/src/main/resources/twitter-server/js/histogram-dom.js b/server/src/main/resources/twitter-server/js/histogram-dom.js index f18f6d20..c62cef8f 100644 --- a/server/src/main/resources/twitter-server/js/histogram-dom.js +++ b/server/src/main/resources/twitter-server/js/histogram-dom.js @@ -143,7 +143,7 @@ function refreshHistogram(updatedValues) { addColumns(); for (var i = 0; i < updatedValues.length; i++) { var p = updatedValues[i]; - var midpoint = (p.lowerLimit + p.upperLimit) / 2; + var midpoint = (p.lower_limit + p.upper_limit) / 2; data.addRow([midpoint, p.percentage]); } } From 5dcbf3a82fc2052518a88dc7b38b68b57cff6c17 Mon Sep 17 00:00:00 2001 From: Matt Dannenberg Date: Wed, 19 May 2021 17:52:03 +0000 Subject: [PATCH 253/409] util-stats: store expressions indexed via a new caseclass instead of string Problem Expressions are stored with a brittle String key. Leading to cases where expressions may clobber one another unexpectedly. Solution Swap the String key (which contains the name and serviceName of the expression) for a caseclass (composed of the name, serviceName, and namespaces of the expression). JIRA Issues: CSL-10620 Differential Revision: https://phabricator.twitter.biz/D666502 --- .../MetricMetadataQueryHandlerTest.scala | 8 +++--- .../exp/MetricExpressionHandlerTest.scala | 25 +++++++++++++------ .../server/util/MetricSchemaSourceTest.scala | 6 ++--- 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala index 289a2682..ca2b2c94 100644 --- a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala @@ -2,7 +2,7 @@ package com.twitter.server.handler import com.twitter.finagle.http.Request import com.twitter.finagle.stats._ -import com.twitter.finagle.stats.exp.ExpressionSchema +import com.twitter.finagle.stats.exp.{ExpressionSchema, ExpressionSchemaKey} import com.twitter.server.util.{JsonUtils, MetricSchemaSource} import com.twitter.util.{Await, Duration} import org.scalatest.FunSuite @@ -60,13 +60,13 @@ class MetricMetadataQueryHandlerTest extends FunSuite { trait UnlatchedRegistry extends SchemaRegistry { val hasLatchedCounters = false def schemas(): Map[String, MetricSchema] = schemaMap - def expressions(): Map[String, ExpressionSchema] = Map.empty + def expressions(): Map[ExpressionSchemaKey, ExpressionSchema] = Map.empty } trait LatchedRegistry extends SchemaRegistry { val hasLatchedCounters = true override def schemas(): Map[String, MetricSchema] = schemaMap - def expressions(): Map[String, ExpressionSchema] = Map.empty + def expressions(): Map[ExpressionSchemaKey, ExpressionSchema] = Map.empty } val latchedSchemaRegistry = new LatchedRegistry() {} @@ -387,7 +387,7 @@ class MetricMetadataQueryHandlerTest extends FunSuite { */ trait SchemalessRegistry extends SchemaRegistry { val hasLatchedCounters = false - def expressions(): Map[String, ExpressionSchema] = Map.empty + def expressions(): Map[ExpressionSchemaKey, ExpressionSchema] = Map.empty } def histoSuffixTestCase( diff --git a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala index 86cad593..75cbc953 100644 --- a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala @@ -2,7 +2,13 @@ package com.twitter.server.handler.exp import com.twitter.conversions.DurationOps._ import com.twitter.finagle.http.Request -import com.twitter.finagle.stats.exp.{Expression, ExpressionSchema, GreaterThan, MonotoneThresholds} +import com.twitter.finagle.stats.exp.{ + Expression, + ExpressionSchema, + ExpressionSchemaKey, + GreaterThan, + MonotoneThresholds +} import com.twitter.finagle.stats.{ CounterSchema, GaugeSchema, @@ -43,16 +49,20 @@ class MetricExpressionHandlerTest extends FunSuite { val latencyP99 = ExpressionSchema("latency_p99", Expression(latencyMb, Right(0.99))).withNamespaces("tenantName") - val expressionSchemaMap: Map[String, ExpressionSchema] = Map( - "success_rate" -> successRateExpression, - "throughput" -> throughputExpression, - "latency" -> latencyP99 + val expressionSchemaMap: Map[ExpressionSchemaKey, ExpressionSchema] = Map( + ExpressionSchemaKey("success_rate", None, Seq()) -> successRateExpression, + ExpressionSchemaKey("throughput", None, Seq()) -> throughputExpression, + ExpressionSchemaKey("latency", None, Seq("path", "to", "tenantName")) -> latencyP99 ) val expressionRegistry = new SchemaRegistry { def hasLatchedCounters: Boolean = false def schemas(): Map[String, MetricSchema] = Map.empty - val expressions: Map[String, ExpressionSchema] = expressionSchemaMap + + val expressions: _root_.scala.Predef.Map[ + _root_.com.twitter.finagle.stats.exp.ExpressionSchemaKey, + _root_.com.twitter.finagle.stats.exp.ExpressionSchema + ] = expressionSchemaMap } val expressionSource = new MetricSchemaSource(Seq(expressionRegistry)) val expressionHandler = new MetricExpressionHandler(expressionSource) @@ -60,7 +70,8 @@ class MetricExpressionHandlerTest extends FunSuite { val latchedRegistry = new SchemaRegistry { def hasLatchedCounters: Boolean = true def schemas(): Map[String, MetricSchema] = Map.empty - def expressions(): Map[String, ExpressionSchema] = expressionSchemaMap + def expressions(): Map[ExpressionSchemaKey, ExpressionSchema] = + expressionSchemaMap } val latchedSource = new MetricSchemaSource(Seq(latchedRegistry)) val latchedHandler = new MetricExpressionHandler(latchedSource) diff --git a/server/src/test/scala/com/twitter/server/util/MetricSchemaSourceTest.scala b/server/src/test/scala/com/twitter/server/util/MetricSchemaSourceTest.scala index 8f001878..469105c3 100644 --- a/server/src/test/scala/com/twitter/server/util/MetricSchemaSourceTest.scala +++ b/server/src/test/scala/com/twitter/server/util/MetricSchemaSourceTest.scala @@ -1,7 +1,7 @@ package com.twitter.server.util import com.twitter.finagle.stats._ -import com.twitter.finagle.stats.exp.ExpressionSchema +import com.twitter.finagle.stats.exp.{ExpressionSchema, ExpressionSchemaKey} import org.scalatest.FunSuite class MetricSchemaSourceTest extends FunSuite { @@ -44,13 +44,13 @@ class MetricSchemaSourceTest extends FunSuite { private val latchedPopulatedRegistry: SchemaRegistry = new SchemaRegistry { def hasLatchedCounters: Boolean = true def schemas(): Map[String, MetricSchema] = schemaMap - def expressions(): Map[String, ExpressionSchema] = Map.empty + def expressions(): Map[ExpressionSchemaKey, ExpressionSchema] = Map.empty } private val unlatchedPopulatedRegistry: SchemaRegistry = new SchemaRegistry { def hasLatchedCounters: Boolean = false def schemas(): Map[String, MetricSchema] = schemaMap - def expressions(): Map[String, ExpressionSchema] = Map.empty + def expressions(): Map[ExpressionSchemaKey, ExpressionSchema] = Map.empty } private val metricSchemaSource = new MetricSchemaSource(Seq(latchedPopulatedRegistry)) From 1c82625e68a1a0785a9a0399bcf0df94cb6cfa5a Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Thu, 20 May 2021 20:16:10 +0000 Subject: [PATCH 254/409] Tests: Migrate to AnyFunSuite Problem FunSuite is deprecated for a while, and most importantly, there are so many warning messages in the sbt build output. Solution Migrate them to AnyFunSuite via autofix: https://github.com/scalatest/autofix/tree/master/3.1.x Differential Revision: https://phabricator.twitter.biz/D672801 --- .../server/handler/logback/classic/LoggingHandlerTest.scala | 4 ++-- .../test/scala/com/twitter/server/AdminHttpServerTest.scala | 4 ++-- .../scala/com/twitter/server/ExportingHistogramsTest.scala | 4 ++-- .../scala/com/twitter/server/HistogramQueryHandlerTest.scala | 4 ++-- server/src/test/scala/com/twitter/server/HookTest.scala | 4 ++-- server/src/test/scala/com/twitter/server/LintersTest.scala | 4 ++-- .../test/scala/com/twitter/server/PromoteToOldGenTest.scala | 4 ++-- server/src/test/scala/com/twitter/server/ResolverTest.scala | 4 ++-- .../test/scala/com/twitter/server/TwitterServerTest.scala | 4 ++-- .../twitter/server/filters/AdminThreadPoolFilterTest.scala | 4 ++-- .../twitter/server/handler/AttachedClientsHandlerTest.scala | 4 ++-- .../twitter/server/handler/ClientRegistryHandlerTest.scala | 4 ++-- .../scala/com/twitter/server/handler/LintHandlerTest.scala | 4 ++-- .../server/handler/MetricMetadataQueryHandlerTest.scala | 4 ++-- .../twitter/server/handler/MetricTypeQueryHandlerTest.scala | 4 ++-- .../com/twitter/server/handler/NoLoggingHandlerTest.scala | 4 ++-- .../com/twitter/server/handler/RegistryHandlerTest.scala | 4 ++-- .../com/twitter/server/handler/ResourceHandlerTest.scala | 4 ++-- .../com/twitter/server/handler/ServerInfoHandlerTest.scala | 4 ++-- .../twitter/server/handler/ServerRegistryHandlerTest.scala | 4 ++-- .../com/twitter/server/handler/ShutdownHandlerTest.scala | 4 ++-- .../com/twitter/server/handler/ThreadsHandlerTest.scala | 4 ++-- .../scala/com/twitter/server/handler/ToggleHandlerTest.scala | 4 ++-- .../com/twitter/server/handler/TracingHandlerTest.scala | 5 +++-- .../com/twitter/server/handler/TunableHandlerTest.scala | 4 ++-- .../com/twitter/server/handler/exp/ExpressionSerdeTest.scala | 4 ++-- .../server/handler/exp/MetricExpressionHandlerTest.scala | 4 ++-- .../twitter/server/lint/DuplicateFlagDefinitionsTest.scala | 4 ++-- .../server/lint/DuplicateLoadServiceBindingsTest.scala | 4 ++-- .../scala/com/twitter/server/lint/LoggingRulesTest.scala | 4 ++-- .../com/twitter/server/lint/MemcacheFailFastRuleTest.scala | 4 ++-- .../com/twitter/server/lint/NullStatsReceiversRuleTest.scala | 4 ++-- .../twitter/server/lint/NumberOfStatsReceiversRuleTest.scala | 4 ++-- .../com/twitter/server/lint/SchedulerBlockingRuleTest.scala | 5 +++-- .../server/lint/StackRegistryDuplicatesRuleTest.scala | 4 ++-- .../test/scala/com/twitter/server/util/HtmlUtilsTest.scala | 4 ++-- .../test/scala/com/twitter/server/util/HttpUtilsTest.scala | 4 ++-- .../com/twitter/server/util/MetricSchemaJsonModuleTest.scala | 4 ++-- .../com/twitter/server/util/MetricSchemaSourceTest.scala | 4 ++-- .../scala/com/twitter/server/util/MetricSourceTest.scala | 4 ++-- .../scala/com/twitter/server/view/CompositeViewTest.scala | 4 ++-- .../test/scala/com/twitter/server/view/IndexViewTest.scala | 4 ++-- .../scala/com/twitter/server/view/NotFoundViewTest.scala | 4 ++-- .../com/twitter/server/view/StackRegistryViewTest.scala | 4 ++-- .../scala/com/twitter/server/view/TextBlockViewTest.scala | 4 ++-- server/src/test/scala/com/twitter/server/view/ViewTest.scala | 4 ++-- .../src/test/scala/com/twitter/server/HandlersTest.scala | 4 ++-- .../src/test/scala/com/twitter/server/LogFormatTest.scala | 4 ++-- .../server/handler/slf4j/log4j12/LoggingHandlerTest.scala | 4 ++-- 49 files changed, 100 insertions(+), 98 deletions(-) diff --git a/logback-classic/src/test/scala/com/twitter/server/handler/logback/classic/LoggingHandlerTest.scala b/logback-classic/src/test/scala/com/twitter/server/handler/logback/classic/LoggingHandlerTest.scala index a7bd6ed6..442da084 100644 --- a/logback-classic/src/test/scala/com/twitter/server/handler/logback/classic/LoggingHandlerTest.scala +++ b/logback-classic/src/test/scala/com/twitter/server/handler/logback/classic/LoggingHandlerTest.scala @@ -5,10 +5,10 @@ import com.twitter.conversions.DurationOps._ import com.twitter.finagle.http.Request import com.twitter.util.Await import com.twitter.util.logging.{Logger => UtilLogger} -import org.scalatest.FunSuite import org.slf4j.LoggerFactory +import org.scalatest.funsuite.AnyFunSuite -class LoggingHandlerTest extends FunSuite { +class LoggingHandlerTest extends AnyFunSuite { private val handler = new LoggingHandler() diff --git a/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala b/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala index e8871553..8dbc4c4e 100644 --- a/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala +++ b/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala @@ -6,8 +6,8 @@ import com.twitter.finagle.http._ import com.twitter.server.util.HttpUtils._ import com.twitter.util._ import java.net.InetSocketAddress -import org.scalatest.FunSuite import org.scalatest.concurrent.{Eventually, IntegrationPatience} +import org.scalatest.funsuite.AnyFunSuite class MockMetricsExporter extends HttpMuxHandler { val pattern = "/admin/metrics.json" @@ -55,7 +55,7 @@ class MockClosableHandler extends HttpMuxHandler { } } -class AdminHttpServerTest extends FunSuite with Eventually with IntegrationPatience { +class AdminHttpServerTest extends AnyFunSuite with Eventually with IntegrationPatience { def checkServer(server: ListeningServer, shadow: Boolean = false): Unit = { val port = server.boundAddress.asInstanceOf[InetSocketAddress].getPort diff --git a/server/src/test/scala/com/twitter/server/ExportingHistogramsTest.scala b/server/src/test/scala/com/twitter/server/ExportingHistogramsTest.scala index 9289fe30..967a36c8 100644 --- a/server/src/test/scala/com/twitter/server/ExportingHistogramsTest.scala +++ b/server/src/test/scala/com/twitter/server/ExportingHistogramsTest.scala @@ -1,9 +1,9 @@ package com.twitter.server import com.twitter.finagle.stats.BucketAndCount -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class ExportingHistogramsTest extends FunSuite { +class ExportingHistogramsTest extends AnyFunSuite { import handler.HistogramQueryHandler.{pdf, cdf, BucketAndPercentage} val empty = Seq.empty diff --git a/server/src/test/scala/com/twitter/server/HistogramQueryHandlerTest.scala b/server/src/test/scala/com/twitter/server/HistogramQueryHandlerTest.scala index 5227bc9b..ae251742 100644 --- a/server/src/test/scala/com/twitter/server/HistogramQueryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/HistogramQueryHandlerTest.scala @@ -8,9 +8,9 @@ import com.twitter.finagle.http.{Request, Response} import com.twitter.finagle.stats.{BucketAndCount, InMemoryStatsReceiver} import com.twitter.server.handler.HistogramQueryHandler import com.twitter.util.{Await, Future} -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class HistogramQueryHandlerTest extends FunSuite { +class HistogramQueryHandlerTest extends AnyFunSuite { def await[A](f: Future[A]): A = Await.result(f, 5.seconds) private[this] val mapper = new ObjectMapper with ScalaObjectMapper { registerModule(DefaultScalaModule) diff --git a/server/src/test/scala/com/twitter/server/HookTest.scala b/server/src/test/scala/com/twitter/server/HookTest.scala index 36fe3ee1..cb57c4b7 100644 --- a/server/src/test/scala/com/twitter/server/HookTest.scala +++ b/server/src/test/scala/com/twitter/server/HookTest.scala @@ -2,8 +2,8 @@ package com.twitter.server import com.twitter.conversions.DurationOps._ import com.twitter.util.Duration -import org.scalatest.FunSuite import scala.collection.mutable +import org.scalatest.funsuite.AnyFunSuite class OnExitHook1 extends Hook { override def onExit(): Unit = { @@ -54,7 +54,7 @@ class HookTestTwitterServer(hooks: Seq[Hook]) extends TwitterServer { } } -class HookTest extends FunSuite { +class HookTest extends AnyFunSuite { val hooks: Seq[Hook] = Seq(new OnExitHook1(), new OnExitHook2()) diff --git a/server/src/test/scala/com/twitter/server/LintersTest.scala b/server/src/test/scala/com/twitter/server/LintersTest.scala index 0a6e6333..7d34c4fd 100644 --- a/server/src/test/scala/com/twitter/server/LintersTest.scala +++ b/server/src/test/scala/com/twitter/server/LintersTest.scala @@ -1,8 +1,8 @@ package com.twitter.server -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class LintersTest extends FunSuite { +class LintersTest extends AnyFunSuite { val server = new TestTwitterServer val ruleIdsSet = server.linterRules.map(_.id).toSet diff --git a/server/src/test/scala/com/twitter/server/PromoteToOldGenTest.scala b/server/src/test/scala/com/twitter/server/PromoteToOldGenTest.scala index 6388b421..177f8f68 100644 --- a/server/src/test/scala/com/twitter/server/PromoteToOldGenTest.scala +++ b/server/src/test/scala/com/twitter/server/PromoteToOldGenTest.scala @@ -1,8 +1,8 @@ package com.twitter.server -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class PromoteToOldGenTest extends FunSuite { +class PromoteToOldGenTest extends AnyFunSuite { test("beforeServing does nothing if flag is disabled") { val promote = new Lifecycle.PromoteToOldGen(Nil) diff --git a/server/src/test/scala/com/twitter/server/ResolverTest.scala b/server/src/test/scala/com/twitter/server/ResolverTest.scala index 8c63c635..8c0dc692 100644 --- a/server/src/test/scala/com/twitter/server/ResolverTest.scala +++ b/server/src/test/scala/com/twitter/server/ResolverTest.scala @@ -4,7 +4,7 @@ import com.twitter.conversions.DurationOps._ import com.twitter.finagle.{Announcer, Announcement, Resolver} import com.twitter.util.{Await, Awaitable, Future} import java.net.{InetAddress, InetSocketAddress} -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite case class TestAnnouncement(addr: InetSocketAddress, target: String) extends Announcement { def unannounce() = Future.Done @@ -17,7 +17,7 @@ class TestAnnouncer extends Announcer { } // Called InternalResolverTest to avoid conflict with twitter-server -class ResolverTest extends FunSuite { +class ResolverTest extends AnyFunSuite { private[this] def await[T](a: Awaitable[T]): T = Await.result(a, 2.seconds) diff --git a/server/src/test/scala/com/twitter/server/TwitterServerTest.scala b/server/src/test/scala/com/twitter/server/TwitterServerTest.scala index 900ad1b0..b79ac06c 100644 --- a/server/src/test/scala/com/twitter/server/TwitterServerTest.scala +++ b/server/src/test/scala/com/twitter/server/TwitterServerTest.scala @@ -6,9 +6,9 @@ import com.twitter.finagle.Service import com.twitter.finagle.http.{Request, Response} import com.twitter.util._ import java.util.concurrent.Executors -import org.scalatest.FunSuite import scala.collection.mutable import scala.util.control.NonFatal +import org.scalatest.funsuite.AnyFunSuite class TestTwitterServer(await: Boolean = false) extends TwitterServer { override val defaultAdminPort = 0 @@ -54,7 +54,7 @@ class MockExceptionHandler extends Service[Request, Response] { throw new Exception("test exception") } } -class TwitterServerTest extends FunSuite { +class TwitterServerTest extends AnyFunSuite { test("TwitterServer does not prematurely execute lifecycle hooks") { val twitterServer = new TestTwitterServer() assert(twitterServer.bootstrapSeq.isEmpty) diff --git a/server/src/test/scala/com/twitter/server/filters/AdminThreadPoolFilterTest.scala b/server/src/test/scala/com/twitter/server/filters/AdminThreadPoolFilterTest.scala index 8649936c..71b5de3f 100644 --- a/server/src/test/scala/com/twitter/server/filters/AdminThreadPoolFilterTest.scala +++ b/server/src/test/scala/com/twitter/server/filters/AdminThreadPoolFilterTest.scala @@ -4,9 +4,9 @@ import com.twitter.conversions.DurationOps._ import com.twitter.finagle.Service import com.twitter.finagle.http.{Request, Response} import com.twitter.util.{Await, Future} -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class AdminThreadPoolFilterTest extends FunSuite { +class AdminThreadPoolFilterTest extends AnyFunSuite { test("work done in the dedicated admin thread pool") { @volatile var workingThreadId: Long = -1 diff --git a/server/src/test/scala/com/twitter/server/handler/AttachedClientsHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/AttachedClientsHandlerTest.scala index c8860994..7b405b7a 100644 --- a/server/src/test/scala/com/twitter/server/handler/AttachedClientsHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/AttachedClientsHandlerTest.scala @@ -14,7 +14,7 @@ import java.security.cert.X509Certificate import java.util.Date import javax.net.ssl.SSLSession import org.mockito.Mockito -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite object AttachedClientsHandlerTest { def await[A](f: Future[A]): A = Await.result(f, 2.seconds) @@ -59,7 +59,7 @@ object AttachedClientsHandlerTest { } } -class AttachedClientsHandlerTest extends FunSuite { +class AttachedClientsHandlerTest extends AnyFunSuite { import AttachedClientsHandlerTest._ private[this] def assertJsonResponse(actualResponse: String, expectedResponse: String) = { diff --git a/server/src/test/scala/com/twitter/server/handler/ClientRegistryHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/ClientRegistryHandlerTest.scala index 958ca79f..f15b889b 100644 --- a/server/src/test/scala/com/twitter/server/handler/ClientRegistryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/ClientRegistryHandlerTest.scala @@ -9,9 +9,9 @@ import com.twitter.server.util.MetricSourceTest import com.twitter.util.{Await, Awaitable, Time} import java.net.URLEncoder import java.nio.charset.StandardCharsets -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class ClientRegistryHandlerTest extends FunSuite { +class ClientRegistryHandlerTest extends AnyFunSuite { private[this] def await[T](a: Awaitable[T]): T = Await.result(a, 2.seconds) diff --git a/server/src/test/scala/com/twitter/server/handler/LintHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/LintHandlerTest.scala index cdf1a32c..db930975 100644 --- a/server/src/test/scala/com/twitter/server/handler/LintHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/LintHandlerTest.scala @@ -4,9 +4,9 @@ import com.twitter.conversions.DurationOps._ import com.twitter.finagle.http.Request import com.twitter.util.Await import com.twitter.util.lint.{Category, Issue, Rule} -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class LintHandlerTest extends FunSuite { +class LintHandlerTest extends AnyFunSuite { private val ruleOk = Rule(Category.Performance, "ok", "desc") { Nil } private def nFailed(n: Int): Rule = Rule(Category.Performance, "failures", "desc") { diff --git a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala index ca2b2c94..381550e0 100644 --- a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala @@ -5,9 +5,9 @@ import com.twitter.finagle.stats._ import com.twitter.finagle.stats.exp.{ExpressionSchema, ExpressionSchemaKey} import com.twitter.server.util.{JsonUtils, MetricSchemaSource} import com.twitter.util.{Await, Duration} -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class MetricMetadataQueryHandlerTest extends FunSuite { +class MetricMetadataQueryHandlerTest extends AnyFunSuite { val schemaMap: Map[String, MetricSchema] = Map( "my/cool/counter" -> CounterSchema( diff --git a/server/src/test/scala/com/twitter/server/handler/MetricTypeQueryHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/MetricTypeQueryHandlerTest.scala index 21199593..86d450b4 100644 --- a/server/src/test/scala/com/twitter/server/handler/MetricTypeQueryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/MetricTypeQueryHandlerTest.scala @@ -8,9 +8,9 @@ import java.util.concurrent.TimeUnit import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.scala.{DefaultScalaModule, ScalaObjectMapper} -import org.scalatest.FunSuite import scala.collection.mutable +import org.scalatest.funsuite.AnyFunSuite object MetricTypeQueryHandlerTest { // Needs to be in companion object because of Jackson. We use `Set` to avoid ordering issues here. @@ -18,7 +18,7 @@ object MetricTypeQueryHandlerTest { case class Metric(name: String, kind: String) } -class MetricTypeQueryHandlerTest extends FunSuite { +class MetricTypeQueryHandlerTest extends AnyFunSuite { import MetricTypeQueryHandlerTest._ private[this] val mapper = new ObjectMapper with ScalaObjectMapper { diff --git a/server/src/test/scala/com/twitter/server/handler/NoLoggingHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/NoLoggingHandlerTest.scala index c3ecd8a6..8e97df59 100644 --- a/server/src/test/scala/com/twitter/server/handler/NoLoggingHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/NoLoggingHandlerTest.scala @@ -3,9 +3,9 @@ package com.twitter.server.handler import com.twitter.conversions.DurationOps._ import com.twitter.finagle.http.{Request, Status} import com.twitter.util.Await -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class NoLoggingHandlerTest extends FunSuite { +class NoLoggingHandlerTest extends AnyFunSuite { test("NoLoggingHandler responds with correct message") { val handler = new NoLoggingHandler diff --git a/server/src/test/scala/com/twitter/server/handler/RegistryHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/RegistryHandlerTest.scala index a8cd84b0..93348c0a 100644 --- a/server/src/test/scala/com/twitter/server/handler/RegistryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/RegistryHandlerTest.scala @@ -3,9 +3,9 @@ package com.twitter.server.handler import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.scala.{DefaultScalaModule, ScalaObjectMapper} import com.twitter.util.registry.{GlobalRegistry, SimpleRegistry} -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class RegistryHandlerTest extends FunSuite { +class RegistryHandlerTest extends AnyFunSuite { private[this] val mapper = new ObjectMapper with ScalaObjectMapper { registerModule(DefaultScalaModule) diff --git a/server/src/test/scala/com/twitter/server/handler/ResourceHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/ResourceHandlerTest.scala index b3d63aca..30846abb 100644 --- a/server/src/test/scala/com/twitter/server/handler/ResourceHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/ResourceHandlerTest.scala @@ -7,10 +7,10 @@ import com.twitter.util.{Await, Awaitable} import java.io.{ByteArrayInputStream, File, FileWriter, InputStream} import java.nio.charset.StandardCharsets.ISO_8859_1 import java.nio.file.Files -import org.scalatest.FunSuite import scala.io.Source +import org.scalatest.funsuite.AnyFunSuite -class ResourceHandlerTest extends FunSuite { +class ResourceHandlerTest extends AnyFunSuite { private[this] def await[T](a: Awaitable[T]): T = Await.result(a, 2.seconds) diff --git a/server/src/test/scala/com/twitter/server/handler/ServerInfoHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/ServerInfoHandlerTest.scala index b2444555..61dd67a8 100644 --- a/server/src/test/scala/com/twitter/server/handler/ServerInfoHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/ServerInfoHandlerTest.scala @@ -4,9 +4,9 @@ import com.twitter.conversions.DurationOps._ import com.twitter.finagle.http.{Request, Status} import com.twitter.util.registry.{GlobalRegistry, SimpleRegistry} import com.twitter.util.{Await, Awaitable} -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class ServerInfoHandlerTest extends FunSuite { +class ServerInfoHandlerTest extends AnyFunSuite { private[this] def await[T](a: Awaitable[T]): T = Await.result(a, 2.seconds) diff --git a/server/src/test/scala/com/twitter/server/handler/ServerRegistryHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/ServerRegistryHandlerTest.scala index f45525e6..6816c88b 100644 --- a/server/src/test/scala/com/twitter/server/handler/ServerRegistryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/ServerRegistryHandlerTest.scala @@ -8,9 +8,9 @@ import com.twitter.server.util.MetricSourceTest import com.twitter.util.{Await, Awaitable} import java.net.URLEncoder import java.nio.charset.StandardCharsets -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class ServerRegistryHandlerTest extends FunSuite { +class ServerRegistryHandlerTest extends AnyFunSuite { private[this] def await[T](a: Awaitable[T]): T = Await.result(a, 2.seconds) diff --git a/server/src/test/scala/com/twitter/server/handler/ShutdownHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/ShutdownHandlerTest.scala index 7fd0ac04..abe9fcbf 100644 --- a/server/src/test/scala/com/twitter/server/handler/ShutdownHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/ShutdownHandlerTest.scala @@ -4,9 +4,9 @@ import com.twitter.conversions.DurationOps._ import com.twitter.finagle.http.{Status, Request, Method} import com.twitter.server.TwitterServer import com.twitter.util.{Await, Awaitable, Closable, Future, Time} -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class ShutdownHandlerTest extends FunSuite { +class ShutdownHandlerTest extends AnyFunSuite { private[this] def await[T](a: Awaitable[T]): T = Await.result(a, 2.seconds) diff --git a/server/src/test/scala/com/twitter/server/handler/ThreadsHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/ThreadsHandlerTest.scala index 78613f06..1bdaf5d3 100644 --- a/server/src/test/scala/com/twitter/server/handler/ThreadsHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/ThreadsHandlerTest.scala @@ -5,9 +5,9 @@ import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.scala.DefaultScalaModule import com.twitter.finagle.http.Request import com.twitter.util.{Await, Awaitable} -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class ThreadsHandlerTest extends FunSuite { +class ThreadsHandlerTest extends AnyFunSuite { type Threads = Map[String, Map[String, Map[String, Any]]] private[this] def await[T](a: Awaitable[T]): T = Await.result(a, 2.seconds) diff --git a/server/src/test/scala/com/twitter/server/handler/ToggleHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/ToggleHandlerTest.scala index eaae6f0f..06cec504 100644 --- a/server/src/test/scala/com/twitter/server/handler/ToggleHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/ToggleHandlerTest.scala @@ -3,11 +3,11 @@ package com.twitter.server.handler import com.twitter.finagle.stats.NullStatsReceiver import com.twitter.finagle.toggle.{Toggle, ToggleMap} import com.twitter.server.handler.ToggleHandler._ -import org.scalatest.FunSuite import scala.collection.immutable import scala.collection.mutable.ArrayBuffer +import org.scalatest.funsuite.AnyFunSuite -class ToggleHandlerTest extends FunSuite { +class ToggleHandlerTest extends AnyFunSuite { test("renders empty registeredLibraries") { val handler = new ToggleHandler(() => Map.empty) diff --git a/server/src/test/scala/com/twitter/server/handler/TracingHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/TracingHandlerTest.scala index 56c384d1..40de77c5 100644 --- a/server/src/test/scala/com/twitter/server/handler/TracingHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/TracingHandlerTest.scala @@ -4,10 +4,11 @@ import com.twitter.conversions.DurationOps._ import com.twitter.finagle.http.{Request, Status} import com.twitter.finagle.tracing.{Trace, Tracer} import com.twitter.util.{Await, Awaitable} -import org.scalatest.{BeforeAndAfter, FunSuite} +import org.scalatest.BeforeAndAfter import org.scalatestplus.mockito.MockitoSugar +import org.scalatest.funsuite.AnyFunSuite -class TracingHandlerTest extends FunSuite with MockitoSugar with BeforeAndAfter { +class TracingHandlerTest extends AnyFunSuite with MockitoSugar with BeforeAndAfter { val service = new TracingHandler private[this] def await[T](a: Awaitable[T]): T = Await.result(a, 2.seconds) diff --git a/server/src/test/scala/com/twitter/server/handler/TunableHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/TunableHandlerTest.scala index 9c1985e6..77cad562 100644 --- a/server/src/test/scala/com/twitter/server/handler/TunableHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/TunableHandlerTest.scala @@ -6,7 +6,7 @@ import com.twitter.conversions.DurationOps._ import com.twitter.finagle.http.{MediaType, Method, Request, Status} import com.twitter.util.Await import com.twitter.util.tunable.TunableMap -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite object TunableHandlerTest { // Needs to be in companion object because of Jackson. We use `Set` to avoid ordering issues here. @@ -15,7 +15,7 @@ object TunableHandlerTest { case class Component(source: String, value: String) } -class TunableHandlerTest extends FunSuite { +class TunableHandlerTest extends AnyFunSuite { import TunableHandlerTest._ private[this] val mapper = new ObjectMapper with ScalaObjectMapper { diff --git a/server/src/test/scala/com/twitter/server/handler/exp/ExpressionSerdeTest.scala b/server/src/test/scala/com/twitter/server/handler/exp/ExpressionSerdeTest.scala index 67fcc5e2..274c51be 100644 --- a/server/src/test/scala/com/twitter/server/handler/exp/ExpressionSerdeTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/exp/ExpressionSerdeTest.scala @@ -9,9 +9,9 @@ import com.twitter.finagle.stats.exp.{ Unbounded } import com.twitter.server.util.{JsonUtils, AdminJsonConverter} -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class ExpressionSerdeTest extends FunSuite { +class ExpressionSerdeTest extends AnyFunSuite { trait BoundsCtx { val unbounded = Unbounded.get val monotoneWithUnset = MonotoneThresholds(GreaterThan, 10, 20) diff --git a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala index 75cbc953..cae3aa28 100644 --- a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala @@ -21,9 +21,9 @@ import com.twitter.finagle.stats.{ import com.twitter.server.handler.MetricExpressionHandler import com.twitter.server.util.{AdminJsonConverter, JsonUtils, MetricSchemaSource} import com.twitter.util.{Await, Awaitable, Duration} -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class MetricExpressionHandlerTest extends FunSuite { +class MetricExpressionHandlerTest extends AnyFunSuite { private[this] def await[T](awaitable: Awaitable[T], timeout: Duration = 5.second): T = Await.result(awaitable, timeout) diff --git a/server/src/test/scala/com/twitter/server/lint/DuplicateFlagDefinitionsTest.scala b/server/src/test/scala/com/twitter/server/lint/DuplicateFlagDefinitionsTest.scala index 32bc87b4..517e32d3 100644 --- a/server/src/test/scala/com/twitter/server/lint/DuplicateFlagDefinitionsTest.scala +++ b/server/src/test/scala/com/twitter/server/lint/DuplicateFlagDefinitionsTest.scala @@ -1,7 +1,7 @@ package com.twitter.server.lint import com.twitter.app.App -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite /** does not have any duplicate flags */ class FlagApp extends App { @@ -31,7 +31,7 @@ class SameFlagDuplicateFlagApp extends App { flag[String]("flag2", "Same flag name, different type.") } -class DuplicateFlagDefinitionsTest extends FunSuite { +class DuplicateFlagDefinitionsTest extends AnyFunSuite { test("no duplicates") { val app = new FlagApp diff --git a/server/src/test/scala/com/twitter/server/lint/DuplicateLoadServiceBindingsTest.scala b/server/src/test/scala/com/twitter/server/lint/DuplicateLoadServiceBindingsTest.scala index 55d78cd3..619b8f87 100644 --- a/server/src/test/scala/com/twitter/server/lint/DuplicateLoadServiceBindingsTest.scala +++ b/server/src/test/scala/com/twitter/server/lint/DuplicateLoadServiceBindingsTest.scala @@ -1,8 +1,8 @@ package com.twitter.server.lint -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class DuplicateLoadServiceBindingsTest extends FunSuite { +class DuplicateLoadServiceBindingsTest extends AnyFunSuite { private trait TestTrait diff --git a/server/src/test/scala/com/twitter/server/lint/LoggingRulesTest.scala b/server/src/test/scala/com/twitter/server/lint/LoggingRulesTest.scala index e33a7fce..27beaa57 100644 --- a/server/src/test/scala/com/twitter/server/lint/LoggingRulesTest.scala +++ b/server/src/test/scala/com/twitter/server/lint/LoggingRulesTest.scala @@ -1,9 +1,9 @@ package com.twitter.server.lint import com.twitter.util.lint.Issue -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class LoggingRulesTest extends FunSuite { +class LoggingRulesTest extends AnyFunSuite { val validUrl = "file:/jars/slf4j-jdk14-1.7.7.jar!/org/slf4j/impl/StaticLoggerBinder.class" val invalidUrl = "/classes/org/slf4j/impl/StaticLoggerBinder.class" diff --git a/server/src/test/scala/com/twitter/server/lint/MemcacheFailFastRuleTest.scala b/server/src/test/scala/com/twitter/server/lint/MemcacheFailFastRuleTest.scala index 08c43403..9ebd8d6a 100644 --- a/server/src/test/scala/com/twitter/server/lint/MemcacheFailFastRuleTest.scala +++ b/server/src/test/scala/com/twitter/server/lint/MemcacheFailFastRuleTest.scala @@ -5,9 +5,9 @@ import com.twitter.finagle.client.StackClient import com.twitter.finagle.param.ProtocolLibrary import com.twitter.finagle.service.FailFastFactory import com.twitter.finagle.util.StackRegistry -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class MemcacheFailFastRuleTest extends FunSuite { +class MemcacheFailFastRuleTest extends AnyFunSuite { def newRegistry(): StackRegistry = new StackRegistry { def registryName: String = "client" } diff --git a/server/src/test/scala/com/twitter/server/lint/NullStatsReceiversRuleTest.scala b/server/src/test/scala/com/twitter/server/lint/NullStatsReceiversRuleTest.scala index 4775c33d..90e2e8a8 100644 --- a/server/src/test/scala/com/twitter/server/lint/NullStatsReceiversRuleTest.scala +++ b/server/src/test/scala/com/twitter/server/lint/NullStatsReceiversRuleTest.scala @@ -5,9 +5,9 @@ import com.twitter.finagle.client.StackClient import com.twitter.finagle.param.{Label, Stats} import com.twitter.finagle.stats.{InMemoryStatsReceiver, NullStatsReceiver} import com.twitter.finagle.util.StackRegistry -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class NullStatsReceiversRuleTest extends FunSuite { +class NullStatsReceiversRuleTest extends AnyFunSuite { def newRegistry(name: String): StackRegistry = new StackRegistry { def registryName: String = name } diff --git a/server/src/test/scala/com/twitter/server/lint/NumberOfStatsReceiversRuleTest.scala b/server/src/test/scala/com/twitter/server/lint/NumberOfStatsReceiversRuleTest.scala index 65131fe5..abb53679 100644 --- a/server/src/test/scala/com/twitter/server/lint/NumberOfStatsReceiversRuleTest.scala +++ b/server/src/test/scala/com/twitter/server/lint/NumberOfStatsReceiversRuleTest.scala @@ -1,9 +1,9 @@ package com.twitter.server.lint import com.twitter.finagle.stats.InMemoryStatsReceiver -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class NumberOfStatsReceiversRuleTest extends FunSuite { +class NumberOfStatsReceiversRuleTest extends AnyFunSuite { test("One stats receiver doesn't create issues") { val statsReceivers = Seq(new InMemoryStatsReceiver) diff --git a/server/src/test/scala/com/twitter/server/lint/SchedulerBlockingRuleTest.scala b/server/src/test/scala/com/twitter/server/lint/SchedulerBlockingRuleTest.scala index 402d28c9..aa3e5da5 100644 --- a/server/src/test/scala/com/twitter/server/lint/SchedulerBlockingRuleTest.scala +++ b/server/src/test/scala/com/twitter/server/lint/SchedulerBlockingRuleTest.scala @@ -3,10 +3,11 @@ package com.twitter.server.lint import com.twitter.concurrent.Scheduler import com.twitter.conversions.DurationOps._ import org.mockito.Mockito._ -import org.scalatest.{FunSuite, Matchers} import org.scalatestplus.mockito.MockitoSugar +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers -class SchedulerBlockingRuleTest extends FunSuite with Matchers with MockitoSugar { +class SchedulerBlockingRuleTest extends AnyFunSuite with Matchers with MockitoSugar { test("low amount of blocking is not an issue") { val cutoff = 10.seconds diff --git a/server/src/test/scala/com/twitter/server/lint/StackRegistryDuplicatesRuleTest.scala b/server/src/test/scala/com/twitter/server/lint/StackRegistryDuplicatesRuleTest.scala index 823dae13..c8962cf9 100644 --- a/server/src/test/scala/com/twitter/server/lint/StackRegistryDuplicatesRuleTest.scala +++ b/server/src/test/scala/com/twitter/server/lint/StackRegistryDuplicatesRuleTest.scala @@ -4,9 +4,9 @@ import com.twitter.finagle.Stack import com.twitter.finagle.client.StackClient import com.twitter.finagle.param.{Label, ProtocolLibrary} import com.twitter.finagle.util.StackRegistry -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class StackRegistryDuplicatesRuleTest extends FunSuite { +class StackRegistryDuplicatesRuleTest extends AnyFunSuite { def newRegistry(): StackRegistry = new StackRegistry { def registryName: String = "client" } diff --git a/server/src/test/scala/com/twitter/server/util/HtmlUtilsTest.scala b/server/src/test/scala/com/twitter/server/util/HtmlUtilsTest.scala index 29941b48..0efefb17 100644 --- a/server/src/test/scala/com/twitter/server/util/HtmlUtilsTest.scala +++ b/server/src/test/scala/com/twitter/server/util/HtmlUtilsTest.scala @@ -1,10 +1,10 @@ package com.twitter.server.util import org.scalacheck.Gen -import org.scalatest.FunSuite import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks +import org.scalatest.funsuite.AnyFunSuite -class HtmlUtilsTest extends FunSuite with ScalaCheckDrivenPropertyChecks { +class HtmlUtilsTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { private val escapedChars = Seq('>', '<', '&', '"', '\'') diff --git a/server/src/test/scala/com/twitter/server/util/HttpUtilsTest.scala b/server/src/test/scala/com/twitter/server/util/HttpUtilsTest.scala index e781f1c0..c6d8deb4 100644 --- a/server/src/test/scala/com/twitter/server/util/HttpUtilsTest.scala +++ b/server/src/test/scala/com/twitter/server/util/HttpUtilsTest.scala @@ -6,9 +6,9 @@ import com.twitter.finagle.http.{HttpMuxer, Request, Response, Status, Version} import com.twitter.io.Buf import com.twitter.server.util.HttpUtils._ import com.twitter.util.{Await, Future, Time} -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class HttpUtilsTest extends FunSuite { +class HttpUtilsTest extends AnyFunSuite { private[this] def await[A](a: Future[A]): A = Await.result(a, 5.seconds) test("combine can combine two muxers") { diff --git a/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala b/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala index 82d118ab..d7cb6232 100644 --- a/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala +++ b/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala @@ -3,9 +3,9 @@ package com.twitter.server.util import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.scala.{DefaultScalaModule, ScalaObjectMapper} import com.twitter.finagle.stats._ -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class MetricSchemaJsonModuleTest extends FunSuite { +class MetricSchemaJsonModuleTest extends AnyFunSuite { private val counterSchema = CounterSchema( MetricBuilder( diff --git a/server/src/test/scala/com/twitter/server/util/MetricSchemaSourceTest.scala b/server/src/test/scala/com/twitter/server/util/MetricSchemaSourceTest.scala index 469105c3..4a54e5f0 100644 --- a/server/src/test/scala/com/twitter/server/util/MetricSchemaSourceTest.scala +++ b/server/src/test/scala/com/twitter/server/util/MetricSchemaSourceTest.scala @@ -2,9 +2,9 @@ package com.twitter.server.util import com.twitter.finagle.stats._ import com.twitter.finagle.stats.exp.{ExpressionSchema, ExpressionSchemaKey} -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class MetricSchemaSourceTest extends FunSuite { +class MetricSchemaSourceTest extends AnyFunSuite { private val schemaMap: Map[String, MetricSchema] = Map( "my/cool/counter" -> CounterSchema( diff --git a/server/src/test/scala/com/twitter/server/util/MetricSourceTest.scala b/server/src/test/scala/com/twitter/server/util/MetricSourceTest.scala index e383ea38..707ddec9 100644 --- a/server/src/test/scala/com/twitter/server/util/MetricSourceTest.scala +++ b/server/src/test/scala/com/twitter/server/util/MetricSourceTest.scala @@ -3,7 +3,7 @@ package com.twitter.server.util import com.twitter.conversions.DurationOps._ import com.twitter.finagle.stats.{StatsRegistry, StatEntry} import com.twitter.util.Time -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite private[server] object MetricSourceTest { class Ctx { @@ -18,7 +18,7 @@ private[server] object MetricSourceTest { } } -class MetricSourceTest extends FunSuite { +class MetricSourceTest extends AnyFunSuite { import MetricSourceTest._ test("get") { diff --git a/server/src/test/scala/com/twitter/server/view/CompositeViewTest.scala b/server/src/test/scala/com/twitter/server/view/CompositeViewTest.scala index dfe61c25..7ce505d1 100644 --- a/server/src/test/scala/com/twitter/server/view/CompositeViewTest.scala +++ b/server/src/test/scala/com/twitter/server/view/CompositeViewTest.scala @@ -1,8 +1,8 @@ package com.twitter.server.view -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class CompositeViewTest extends FunSuite { +class CompositeViewTest extends AnyFunSuite { class CheckView extends View { def render: String = "Check" diff --git a/server/src/test/scala/com/twitter/server/view/IndexViewTest.scala b/server/src/test/scala/com/twitter/server/view/IndexViewTest.scala index 106a8aa4..1287470d 100644 --- a/server/src/test/scala/com/twitter/server/view/IndexViewTest.scala +++ b/server/src/test/scala/com/twitter/server/view/IndexViewTest.scala @@ -6,9 +6,9 @@ import com.twitter.finagle.Service import com.twitter.io.Buf import com.twitter.server.util.HttpUtils.{newOk, newResponse} import com.twitter.util.{Await, Awaitable, Future} -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class IndexViewTest extends FunSuite { +class IndexViewTest extends AnyFunSuite { private[this] def await[T](a: Awaitable[T]): T = Await.result(a, 2.seconds) diff --git a/server/src/test/scala/com/twitter/server/view/NotFoundViewTest.scala b/server/src/test/scala/com/twitter/server/view/NotFoundViewTest.scala index 5d986bfe..4c2cb0c6 100644 --- a/server/src/test/scala/com/twitter/server/view/NotFoundViewTest.scala +++ b/server/src/test/scala/com/twitter/server/view/NotFoundViewTest.scala @@ -6,9 +6,9 @@ import com.twitter.finagle.Service import com.twitter.io.Buf import com.twitter.server.util.HttpUtils.newResponse import com.twitter.util.{Await, Awaitable} -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class NotFoundViewTest extends FunSuite { +class NotFoundViewTest extends AnyFunSuite { private[this] def await[T](a: Awaitable[T]): T = Await.result(a, 2.seconds) diff --git a/server/src/test/scala/com/twitter/server/view/StackRegistryViewTest.scala b/server/src/test/scala/com/twitter/server/view/StackRegistryViewTest.scala index a8dde6bb..d2437b3c 100644 --- a/server/src/test/scala/com/twitter/server/view/StackRegistryViewTest.scala +++ b/server/src/test/scala/com/twitter/server/view/StackRegistryViewTest.scala @@ -3,7 +3,7 @@ package com.twitter.server.view import com.twitter.finagle.param.Label import com.twitter.finagle.util.StackRegistry import com.twitter.finagle.{Stack, StackBuilder, Stackable} -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite private[server] object StackRegistryViewTest { case class Incr(incrementBy: Int) @@ -58,7 +58,7 @@ private[server] object StackRegistryViewTest { val prms = Stack.Params.empty + Label("test") } -class StackRegistryViewTest extends FunSuite { +class StackRegistryViewTest extends AnyFunSuite { import StackRegistryViewTest._ test("render stack") { diff --git a/server/src/test/scala/com/twitter/server/view/TextBlockViewTest.scala b/server/src/test/scala/com/twitter/server/view/TextBlockViewTest.scala index 3acf43fb..d1cc802a 100644 --- a/server/src/test/scala/com/twitter/server/view/TextBlockViewTest.scala +++ b/server/src/test/scala/com/twitter/server/view/TextBlockViewTest.scala @@ -6,9 +6,9 @@ import com.twitter.finagle.http.{Request, Response} import com.twitter.io.Buf import com.twitter.server.util.HttpUtils._ import com.twitter.util.{Await, Awaitable} -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class TextBlockViewTest extends FunSuite { +class TextBlockViewTest extends AnyFunSuite { private[this] def await[T](a: Awaitable[T]): T = Await.result(a, 2.seconds) diff --git a/server/src/test/scala/com/twitter/server/view/ViewTest.scala b/server/src/test/scala/com/twitter/server/view/ViewTest.scala index b2add1bc..95851fa0 100644 --- a/server/src/test/scala/com/twitter/server/view/ViewTest.scala +++ b/server/src/test/scala/com/twitter/server/view/ViewTest.scala @@ -1,8 +1,8 @@ package com.twitter.server.view -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class ViewTest extends FunSuite { +class ViewTest extends AnyFunSuite { class TestView extends View { def render: String = "Hello" diff --git a/slf4j-jdk14/src/test/scala/com/twitter/server/HandlersTest.scala b/slf4j-jdk14/src/test/scala/com/twitter/server/HandlersTest.scala index 47f42990..5a82b96d 100644 --- a/slf4j-jdk14/src/test/scala/com/twitter/server/HandlersTest.scala +++ b/slf4j-jdk14/src/test/scala/com/twitter/server/HandlersTest.scala @@ -6,8 +6,8 @@ import com.twitter.util.{Await, Future} import java.io.ByteArrayOutputStream import java.net.InetSocketAddress import java.util.logging.{Logger, SimpleFormatter, StreamHandler} -import org.scalatest.FunSuite import scala.collection.mutable +import org.scalatest.funsuite.AnyFunSuite /** Test TwitterServer which overrides the admin.port to localhost ephemeral port */ class TestTwitterServer extends TwitterServer { @@ -43,7 +43,7 @@ class MockExceptionHandler extends Service[Request, Response] { } } -class HandlersTest extends FunSuite { +class HandlersTest extends AnyFunSuite { test("Exceptions thrown in handlers include stack traces") { val twitterServer: TwitterServer = new TestTwitterServer { diff --git a/slf4j-jdk14/src/test/scala/com/twitter/server/LogFormatTest.scala b/slf4j-jdk14/src/test/scala/com/twitter/server/LogFormatTest.scala index 9361294f..62be81cc 100644 --- a/slf4j-jdk14/src/test/scala/com/twitter/server/LogFormatTest.scala +++ b/slf4j-jdk14/src/test/scala/com/twitter/server/LogFormatTest.scala @@ -3,9 +3,9 @@ package com.twitter.server import com.twitter.finagle.tracing.{SpanId, Trace, TraceId} import com.twitter.logging.{StringHandler, Level => TwLevel} import java.util.logging.Logger -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class LogFormatTest extends FunSuite { +class LogFormatTest extends AnyFunSuite { def testStringHandler(f: (Logger, StringHandler) => Unit): Unit = { val handler = new StringHandler handler.setFormatter(new com.twitter.server.logging.LogFormatter) diff --git a/slf4j-log4j12/src/test/scala/com/twitter/server/handler/slf4j/log4j12/LoggingHandlerTest.scala b/slf4j-log4j12/src/test/scala/com/twitter/server/handler/slf4j/log4j12/LoggingHandlerTest.scala index 8aecec34..cc76d5bd 100644 --- a/slf4j-log4j12/src/test/scala/com/twitter/server/handler/slf4j/log4j12/LoggingHandlerTest.scala +++ b/slf4j-log4j12/src/test/scala/com/twitter/server/handler/slf4j/log4j12/LoggingHandlerTest.scala @@ -6,10 +6,10 @@ import com.twitter.util.Await import com.twitter.util.logging.{Logger => UtilLogger} import org.apache.log4j.{Level, LogManager} -import org.scalatest.FunSuite import org.slf4j.LoggerFactory +import org.scalatest.funsuite.AnyFunSuite -class LoggingHandlerTest extends FunSuite { +class LoggingHandlerTest extends AnyFunSuite { private val handler = new LoggingHandler() From ec1f12ef9bd118781aa1b6fe38f94dd971f7f1b5 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Fri, 21 May 2021 23:00:28 +0000 Subject: [PATCH 255/409] metrics: Introduce MetricBuilderRegistry for instrumenting expressions in server stack Problem Metrics creations are scattered in many stack modules, each stack module has its filter and independent from others. However, instrumenting an expression needs access to all component metrics. Solution Introduce a stack param to cache metrics as well as providing lazy vals to create expressions. Result The original approach was having a stack.module that sits at the end of the stack where all needed metrics are generated. the downside is it registers the same expressions many times (= connections), and the last override the previous ones. The current approach should reduce the expression evaluation time, but requires implementors to have more context on the finagle stack for metrics creation sequence. Differential Revision: https://phabricator.twitter.biz/D663915 --- .../com/twitter/server/handler/MetricExpressionHandler.scala | 2 +- .../server/handler/exp/MetricExpressionHandlerTest.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala b/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala index 43464be4..da4ad40e 100644 --- a/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala @@ -24,7 +24,7 @@ import com.twitter.server.util.{AdminJsonConverter, MetricSchemaSource} import com.twitter.util.Future object MetricExpressionHandler { - private val Version = 0.7 + private val Version = 1.0 private val statsFormatter = StatsFormatter.default /** diff --git a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala index cae3aa28..f8576f38 100644 --- a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala @@ -95,7 +95,7 @@ class MetricExpressionHandlerTest extends AnyFunSuite { val expectedResponse = """ |{ - | "@version" : 0.7, + | "@version" : 1.0, | "counters_latched" : false, | "separator_char" : "/", | "expressions" : [ From 3a50f94e8f70d81b521fec11c65d65f53d449ab5 Mon Sep 17 00:00:00 2001 From: Lily He Date: Thu, 27 May 2021 15:01:50 +0000 Subject: [PATCH 256/409] Twitter-oss: Prepare OSS libraries for release 21.5.0 Problem We want to release the next versions of our Twitter OSS libraries 21.5.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D677481 --- CHANGELOG.rst | 5 +++++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 53694666..9cd59d36 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +21.5.0 +------ + +No Changes + 21.4.0 ------ diff --git a/README.md b/README.md index 90042271..d4a12216 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index aa87ae6d..b2e22288 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.5.0-SNAPSHOT" +val releaseVersion = "21.5.0" val jacksonVersion = "2.11.2" val jacksonLibs = Seq( From e9fcbdf8eb95249aac62879a9c6dfa293cf1382e Mon Sep 17 00:00:00 2001 From: Lily He Date: Thu, 27 May 2021 16:44:13 +0000 Subject: [PATCH 257/409] Twitter-oss: update OSS libraries post-release to 21.6.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 21.6.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D678026 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d4a12216..90042271 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index b2e22288..74f73596 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.5.0" +val releaseVersion = "21.6.0-SNAPSHOT" val jacksonVersion = "2.11.2" val jacksonLibs = Seq( From a4058b885d433b54411cb2799f809fc48dbbf590 Mon Sep 17 00:00:00 2001 From: Matt Dannenberg Date: Fri, 28 May 2021 19:32:49 +0000 Subject: [PATCH 258/409] twitter-server: add name and namespace params to ExpressionHandler Problem Consumers of the /admin/metric/expressions.json endpoint would like to be able to grab a subset of the expression by name or namespace. There is no way to do so currently. Solution Add optional query params name and namespace which are used to filter the list of expressions returned by the /admin/metric/expressions.json endpoint. JIRA Issues: CSL-10622 Differential Revision: https://phabricator.twitter.biz/D672594 --- .../handler/MetricExpressionHandler.scala | 25 ++- .../server/util/exp/ExpressionJson.scala | 4 +- .../exp/MetricExpressionHandlerTest.scala | 155 +++++++++++++++++- 3 files changed, 176 insertions(+), 8 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala b/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala index da4ad40e..8126857c 100644 --- a/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala @@ -76,11 +76,19 @@ object MetricExpressionHandler { /** * A handler for metric expression queries at admin/metric/expressions.json. + * * @queryParam ?latching_style=boolean Set true to let expression respect the latchedness of counters, * which means it does not wrap the latched counters in `rate()`. - * + * @queryParam ?name=string only return expressions whose name matches the value of name + * @queryParam ?namespace=string only return expressions whose namespace matches the value of namespace. + * note that the namespace is a sequence of strings, which will be joined into a string, + * separated by :s * @example http://$HOST:$PORT/admin/metric/expressions.json?latching_style=true * http://$HOST:$PORT/admin/metric/expressions.json (by default latching_style is false) + * http://$HOST:$PORT/admin/metric/expressions.json?name=success_rate + * (grab all expressions named "success_rate") + * http://$HOST:$PORT/admin/metric/expressions.json?namespace=path:to:namespace + * (grab all expressions of the namespace ["path", "to", "namespace"] */ class MetricExpressionHandler(source: MetricSchemaSource = new MetricSchemaSource) extends Service[Request, Response] { @@ -88,11 +96,20 @@ class MetricExpressionHandler(source: MetricSchemaSource = new MetricSchemaSourc private[this] lazy val sourceLatched = source.hasLatchedCounters def apply(request: Request): Future[Response] = { - val keyParam = Uri.fromRequest(request).params.getAll("latching_style") + val uri = Uri.fromRequest(request) + val latchParam = uri.params.getAll("latching_style") + val nameParam = uri.params.getAll("name").toSet + val namespaceParam = uri.params.getAll("namespace").toSet - val latched = keyParam.exists { value => value == "true" || value == "1" } && sourceLatched + val expressionSchemas = source.expressionList + val filteredSchemas = expressionSchemas.filter(expressionSchema => + // namespace match if namespace param is present + (namespaceParam.isEmpty || namespaceParam.contains(expressionSchema.namespace.mkString(":"))) + // name match if name param is present + && (nameParam.isEmpty || nameParam.contains(expressionSchema.name))) - val expressions = source.expressionList.map { expressionSchema => + val latched = latchParam.exists { value => value == "true" || value == "1" } && sourceLatched + val expressions = filteredSchemas.map { expressionSchema => expressionSchema.copy(exprQuery = translateToQuery(expressionSchema.expr, latched)) } diff --git a/server/src/main/scala/com/twitter/server/util/exp/ExpressionJson.scala b/server/src/main/scala/com/twitter/server/util/exp/ExpressionJson.scala index d60c1ae2..deef21dd 100644 --- a/server/src/main/scala/com/twitter/server/util/exp/ExpressionJson.scala +++ b/server/src/main/scala/com/twitter/server/util/exp/ExpressionJson.scala @@ -35,9 +35,9 @@ object ExpressionJson { gen.writeStringField("role", expressionSchema.labels.role.toString) gen.writeEndObject() - if (expressionSchema.namespaces.nonEmpty) { + if (expressionSchema.namespace.nonEmpty) { gen.writeArrayFieldStart("namespaces") - expressionSchema.namespaces.foreach { p => gen.writeString(p) } + expressionSchema.namespace.foreach { p => gen.writeString(p) } gen.writeEndArray() } diff --git a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala index f8576f38..90ebfcf9 100644 --- a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala @@ -44,10 +44,10 @@ class MetricExpressionHandlerTest extends AnyFunSuite { val throughputExpression = ExpressionSchema("throughput", Expression(successMb).plus(Expression(failuresMb))) - .withNamespaces("path", "to", "tenantName") + .withNamespace("path", "to", "tenantName") val latencyP99 = - ExpressionSchema("latency_p99", Expression(latencyMb, Right(0.99))).withNamespaces("tenantName") + ExpressionSchema("latency_p99", Expression(latencyMb, Right(0.99))).withNamespace("tenantName") val expressionSchemaMap: Map[ExpressionSchemaKey, ExpressionSchema] = Map( ExpressionSchemaKey("success_rate", None, Seq()) -> successRateExpression, @@ -79,6 +79,14 @@ class MetricExpressionHandlerTest extends AnyFunSuite { val testRequest = Request("http://$HOST:$PORT/admin/metric/expressions.json") val latchedStyleRequest = Request( "http://$HOST:$PORT/admin/metric/expressions.json?latching_style=true") + val testRequestWithName = Request( + "http://$HOST:$PORT/admin/metric/expressions.json?name=success_rate") + val testRequestWithNamespace = Request( + "http://$HOST:$PORT/admin/metric/expressions.json?namespace=tenantName") + val testRequestWithNamespacePath = Request( + "http://$HOST:$PORT/admin/metric/expressions.json?namespace=path:to:tenantName") + val testRequestWithNamespaces = Request( + "http://$HOST:$PORT/admin/metric/expressions.json?namespace=path:to:tenantName&namespace=tenantName") private def getSucessRateExpression(json: String): String = { val expressions = @@ -202,4 +210,147 @@ class MetricExpressionHandlerTest extends AnyFunSuite { val result = MetricExpressionHandler.translateToQuery(Expression(connMb)) assert(result == "client/connections") } + + test("Get expression with name param filters to expressions with that name") { + val expectedResponse = + """ + |{ + | "@version" : 1.0, + | "counters_latched" : false, + | "separator_char" : "/", + | "expressions" : [ + | { + | "name" : "success_rate", + | "labels" : { + | "process_path" : "Unspecified", + | "service_name" : "Unspecified", + | "role" : "NoRoleSpecified" + | }, + | "expression" : "multiply(100.0,divide(rate(success),plus(rate(success),rate(failures))))", + | "bounds" : { + | "kind" : "monotone", + | "operator" : ">", + | "bad_threshold" : 99.5, + | "good_threshold" : 99.97, + | "lower_bound_inclusive" : null, + | "upper_bound_exclusive" : null + | }, + | "description" : "Unspecified", + | "unit" : "Unspecified" + | } + | ] + |}""".stripMargin + + JsonUtils.assertJsonResponse( + await(expressionHandler(testRequestWithName)).contentString, + expectedResponse) + } + + test("Get expression with namespace param filters to expressions with that namespace") { + val expectedResponse = + """ + |{ + | "@version" : 1.0, + | "counters_latched" : false, + | "separator_char" : "/", + | "expressions" : [ + | { + | "name" : "throughput", + | "labels" : { + | "process_path" : "Unspecified", + | "service_name" : "Unspecified", + | "role" : "NoRoleSpecified" + | }, + | "namespaces" : ["path","to","tenantName"], + | "expression" : "plus(rate(success),rate(failures))", + | "bounds" : { + | "kind" : "unbounded" + | }, + | "description" : "Unspecified", + | "unit" : "Unspecified" + | }, + | { + | "name" : "latency_p99", + | "labels" : { + | "process_path" : "Unspecified", + | "service_name" : "Unspecified", + | "role" : "NoRoleSpecified" + | }, + | "namespaces" : ["tenantName"], + | "expression" : "latency.p99", + | "bounds" : { + | "kind" : "unbounded" + | }, + | "description" : "Unspecified", + | "unit" : "Unspecified" + | } + | ] + |}""".stripMargin + + JsonUtils.assertJsonResponse( + await(expressionHandler(testRequestWithNamespaces)).contentString, + expectedResponse) + } + + test("Get expression with namespace params filters to expression with that namespace") { + val expectedResponse = + """ + |{ + | "@version" : 1.0, + | "counters_latched" : false, + | "separator_char" : "/", + | "expressions" : [ + | { + | "name" : "latency_p99", + | "labels" : { + | "process_path" : "Unspecified", + | "service_name" : "Unspecified", + | "role" : "NoRoleSpecified" + | }, + | "namespaces" : ["tenantName"], + | "expression" : "latency.p99", + | "bounds" : { + | "kind" : "unbounded" + | }, + | "description" : "Unspecified", + | "unit" : "Unspecified" + | } + | ] + |}""".stripMargin + + JsonUtils.assertJsonResponse( + await(expressionHandler(testRequestWithNamespace)).contentString, + expectedResponse) + } + + test("Get expression with namespace path param filters to expression with that namespace") { + val expectedResponse = + """ + |{ + | "@version" : 1.0, + | "counters_latched" : false, + | "separator_char" : "/", + | "expressions" : [ + | { + | "name" : "throughput", + | "labels" : { + | "process_path" : "Unspecified", + | "service_name" : "Unspecified", + | "role" : "NoRoleSpecified" + | }, + | "namespaces" : ["path","to","tenantName"], + | "expression" : "plus(rate(success),rate(failures))", + | "bounds" : { + | "kind" : "unbounded" + | }, + | "description" : "Unspecified", + | "unit" : "Unspecified" + | } + | ] + |}""".stripMargin + + JsonUtils.assertJsonResponse( + await(expressionHandler(testRequestWithNamespacePath)).contentString, + expectedResponse) + } } From 51b1c7af2b4eede5944897e5c8eec94a36aec325 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Thu, 3 Jun 2021 03:26:30 +0000 Subject: [PATCH 259/409] metrics: Remove MetricSchema and make it an attribute of MetricBuilder Problem The metric types are defined in a wrapper class MetricSchema - CounterSchema, GaugeSchema, and HistogramSchema. This is becoming inconvenient that we need to re-wrap MetricBuilders when we made a change to them. And requires StatsReceiver to carry over the metric type info, while ideally, it should stick with MetricBuilder itself. Solution Remove the wrapper and add enum fields in MetricBuilder. JIRA Issues: CSL-10923 Differential Revision: https://phabricator.twitter.biz/D668739 --- .../handler/MetricExpressionHandler.scala | 21 +++---- .../server/util/MetricSchemaSerializer.scala | 57 ++++++++----------- .../server/util/MetricSchemaSource.scala | 8 +-- .../MetricMetadataQueryHandlerTest.scala | 39 +++++++------ .../exp/MetricExpressionHandlerTest.scala | 24 +++----- .../util/MetricSchemaJsonModuleTest.scala | 20 ++++--- .../server/util/MetricSchemaSourceTest.scala | 22 ++++--- 7 files changed, 93 insertions(+), 98 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala b/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala index 8126857c..a12d8537 100644 --- a/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala @@ -2,6 +2,7 @@ package com.twitter.server.handler import com.twitter.finagle.Service import com.twitter.finagle.http.{MediaType, Request, Response, Uri} +import com.twitter.finagle.stats.MetricBuilder.CounterType import com.twitter.finagle.stats.exp.Expression._ import com.twitter.finagle.stats.exp.{ ConstantExpression, @@ -10,13 +11,7 @@ import com.twitter.finagle.stats.exp.{ HistogramExpression, MetricExpression } -import com.twitter.finagle.stats.{ - CounterSchema, - HistogramSchema, - MetricSchema, - StatsFormatter, - metadataScopeSeparator -} +import com.twitter.finagle.stats.{MetricBuilder, StatsFormatter, metadataScopeSeparator} import com.twitter.io.Buf import com.twitter.server.handler.MetricExpressionHandler.{Version, translateToQuery} import com.twitter.server.util.HttpUtils.newResponse @@ -46,10 +41,10 @@ object MetricExpressionHandler { // the returned metric is styled the same way as admin/metrics.json // e.g.request_latency.p9999 or request_latency.min private def getHisto( - histoSchema: HistogramSchema, + metricBuilder: MetricBuilder, histoComponent: Either[HistogramComponent, Double] ): String = { - val name = histoSchema.metricBuilder.name.mkString(metadataScopeSeparator()) + val name = metricBuilder.name.mkString(metadataScopeSeparator()) val component = histoComponent match { case Right(percentile) => statsFormatter.labelPercentile(percentile) case Left(Min) => statsFormatter.labelMin @@ -63,13 +58,13 @@ object MetricExpressionHandler { // Form metrics other than histograms, rate() for unlatched counters private def getMetric( - metricSchema: MetricSchema, + metricBuilder: MetricBuilder, latched: Boolean, ): String = { - metricSchema match { - case CounterSchema(metricBuilder) if !latched => + metricBuilder.metricType match { + case CounterType if !latched => s"rate(${metricBuilder.name.mkString(metadataScopeSeparator())})" - case other => other.metricBuilder.name.mkString(metadataScopeSeparator()) + case other => metricBuilder.name.mkString(metadataScopeSeparator()) } } } diff --git a/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala b/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala index 14598ed8..65eb6ad9 100644 --- a/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala +++ b/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala @@ -3,16 +3,10 @@ package com.twitter.server.util import com.fasterxml.jackson.core.JsonGenerator import com.fasterxml.jackson.databind.SerializerProvider import com.fasterxml.jackson.databind.ser.std.StdSerializer -import com.twitter.finagle.stats.{ - CounterSchema, - GaugeSchema, - HistogramSchema, - MetricSchema, - StatsFormatter, - metadataScopeSeparator -} +import com.twitter.finagle.stats.MetricBuilder.{CounterType, GaugeType, HistogramType} +import com.twitter.finagle.stats.{MetricBuilder, StatsFormatter, metadataScopeSeparator} -object SchemaSerializer extends StdSerializer[MetricSchema](classOf[MetricSchema]) { +object SchemaSerializer extends StdSerializer[MetricBuilder](classOf[MetricBuilder]) { private[this] val statsFormatter = StatsFormatter.default @@ -29,48 +23,45 @@ object SchemaSerializer extends StdSerializer[MetricSchema](classOf[MetricSchema // 3) The MetricBuilder class would need to be reworked to have a Source case class // nested in it which would contain a few of the currently MetricBuilder level values. def serialize( - metricSchema: MetricSchema, + metricBuilder: MetricBuilder, jsonGenerator: JsonGenerator, serializerProvider: SerializerProvider ): Unit = { - val formattedName = metricSchema.metricBuilder.name.mkString(metadataScopeSeparator()) + val formattedName = metricBuilder.name.mkString(metadataScopeSeparator()) jsonGenerator.writeStartObject() jsonGenerator.writeStringField("name", formattedName) jsonGenerator.writeArrayFieldStart("relative_name") - if (metricSchema.metricBuilder.relativeName != Seq.empty) { - metricSchema.metricBuilder.relativeName.foreach(segment => + if (metricBuilder.relativeName != Seq.empty) { + metricBuilder.relativeName.foreach(segment => jsonGenerator.writeString(convertNullToString(segment))) } else { - metricSchema.metricBuilder.name.foreach(segment => - jsonGenerator.writeString(convertNullToString(segment))) + metricBuilder.name.foreach(segment => jsonGenerator.writeString(convertNullToString(segment))) } jsonGenerator.writeEndArray() - val dataType = metricSchema match { - case _: CounterSchema => "counter" - case _: GaugeSchema => "gauge" - case _: HistogramSchema => "histogram" + val dataType = metricBuilder.metricType match { + case CounterType => "counter" + case GaugeType => "gauge" + case HistogramType => "histogram" } jsonGenerator.writeStringField("kind", dataType) jsonGenerator.writeObjectFieldStart("source") - jsonGenerator.writeStringField( - "class", - metricSchema.metricBuilder.sourceClass.getOrElse("Unspecified")) - jsonGenerator.writeStringField("category", metricSchema.metricBuilder.role.toString) + jsonGenerator.writeStringField("class", metricBuilder.sourceClass.getOrElse("Unspecified")) + jsonGenerator.writeStringField("category", metricBuilder.role.toString) jsonGenerator.writeStringField( "process_path", - metricSchema.metricBuilder.processPath.getOrElse("Unspecified")) + metricBuilder.processPath.getOrElse("Unspecified")) jsonGenerator.writeEndObject() - jsonGenerator.writeStringField("description", metricSchema.metricBuilder.description) - jsonGenerator.writeStringField("unit", metricSchema.metricBuilder.units.toString) - jsonGenerator.writeStringField("verbosity", metricSchema.metricBuilder.verbosity.toString) - jsonGenerator.writeBooleanField("key_indicator", metricSchema.metricBuilder.keyIndicator) + jsonGenerator.writeStringField("description", metricBuilder.description) + jsonGenerator.writeStringField("unit", metricBuilder.units.toString) + jsonGenerator.writeStringField("verbosity", metricBuilder.verbosity.toString) + jsonGenerator.writeBooleanField("key_indicator", metricBuilder.keyIndicator) - metricSchema match { - case _: GaugeSchema => - if (metricSchema.metricBuilder.isCounterishGauge) { + metricBuilder.metricType match { + case GaugeType => + if (metricBuilder.isCounterishGauge) { jsonGenerator.writeBooleanField("counterish_gauge", true) } - case _: HistogramSchema => + case HistogramType => jsonGenerator.writeObjectFieldStart("buckets") jsonGenerator.writeStringField("count", statsFormatter.histogramSeparator + "count") jsonGenerator.writeStringField("sum", statsFormatter.histogramSeparator + "sum") @@ -83,7 +74,7 @@ object SchemaSerializer extends StdSerializer[MetricSchema](classOf[MetricSchema jsonGenerator.writeStringField( "maximum", statsFormatter.histogramSeparator + statsFormatter.labelMax) - metricSchema.metricBuilder.percentiles.foreach(bucket => + metricBuilder.percentiles.foreach(bucket => jsonGenerator.writeStringField( bucket.toString, statsFormatter.histogramSeparator + statsFormatter.labelPercentile(bucket))) diff --git a/server/src/main/scala/com/twitter/server/util/MetricSchemaSource.scala b/server/src/main/scala/com/twitter/server/util/MetricSchemaSource.scala index 076ce151..8fca431e 100644 --- a/server/src/main/scala/com/twitter/server/util/MetricSchemaSource.scala +++ b/server/src/main/scala/com/twitter/server/util/MetricSchemaSource.scala @@ -1,7 +1,7 @@ package com.twitter.server.util import com.twitter.finagle.stats.exp.ExpressionSchema -import com.twitter.finagle.stats.{MetricSchema, SchemaRegistry} +import com.twitter.finagle.stats.{MetricBuilder, SchemaRegistry} import com.twitter.finagle.util.LoadService private[server] object MetricSchemaSource { @@ -26,14 +26,14 @@ private[server] class MetricSchemaSource( } /** Returns the entry for `key` if it exists */ - def getSchema(key: String): Option[MetricSchema] = synchronized { + def getSchema(key: String): Option[MetricBuilder] = synchronized { registry.map(_.schemas()).find(_.contains(key)).flatMap(_.get(key)) } /** Returns all schemas */ - def schemaList(): Iterable[MetricSchema] = synchronized { + def schemaList(): Iterable[MetricBuilder] = synchronized { registry - .foldLeft(IndexedSeq[MetricSchema]()) { (seq, r) => seq ++ r.schemas().values } + .foldLeft(IndexedSeq[MetricBuilder]()) { (seq, r) => seq ++ r.schemas().values } } /** Returns true if the map contains `key` and false otherwise. */ diff --git a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala index 381550e0..090490ad 100644 --- a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala @@ -1,6 +1,7 @@ package com.twitter.server.handler import com.twitter.finagle.http.Request +import com.twitter.finagle.stats.MetricBuilder.{CounterType, GaugeType, HistogramType} import com.twitter.finagle.stats._ import com.twitter.finagle.stats.exp.{ExpressionSchema, ExpressionSchemaKey} import com.twitter.server.util.{JsonUtils, MetricSchemaSource} @@ -9,8 +10,8 @@ import org.scalatest.funsuite.AnyFunSuite class MetricMetadataQueryHandlerTest extends AnyFunSuite { - val schemaMap: Map[String, MetricSchema] = Map( - "my/cool/counter" -> CounterSchema( + val schemaMap: Map[String, MetricBuilder] = Map( + "my/cool/counter" -> MetricBuilder( keyIndicator = true, description = "Counts how many cools are seen", @@ -21,9 +22,10 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { name = Seq("my", "cool", "counter"), processPath = Some("dc/role/zone/service"), percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), + metricType = CounterType, statsReceiver = null - )), - "your/fine/gauge" -> GaugeSchema( + ), + "your/fine/gauge" -> MetricBuilder( keyIndicator = false, description = "Measures how fine the downstream system is", @@ -34,15 +36,17 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { name = Seq("your", "fine", "gauge"), processPath = Some("dc/your_role/zone/your_service"), percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), + metricType = GaugeType, statsReceiver = null - )), - "my/only/histo" -> HistogramSchema( + ), + "my/only/histo" -> MetricBuilder( name = Seq("my", "only", "histo"), percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), + metricType = HistogramType, statsReceiver = null - )), - "my/bad/null/counter" -> CounterSchema( + ), + "my/bad/null/counter" -> MetricBuilder( keyIndicator = true, description = "A counter scoped by null get deserialized correctly", @@ -53,19 +57,20 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { name = Seq("my", "bad", null, "counter"), processPath = Some("dc/role/zone/service"), percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), + metricType = CounterType, statsReceiver = null - )) + ) ) trait UnlatchedRegistry extends SchemaRegistry { val hasLatchedCounters = false - def schemas(): Map[String, MetricSchema] = schemaMap + def schemas(): Map[String, MetricBuilder] = schemaMap def expressions(): Map[ExpressionSchemaKey, ExpressionSchema] = Map.empty } trait LatchedRegistry extends SchemaRegistry { val hasLatchedCounters = true - override def schemas(): Map[String, MetricSchema] = schemaMap + override def schemas(): Map[String, MetricBuilder] = schemaMap def expressions(): Map[ExpressionSchemaKey, ExpressionSchema] = Map.empty } @@ -399,22 +404,24 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { ): Unit = { test(testName) { trait registry extends SchemalessRegistry { - def schemas(): Map[String, MetricSchema] = if (metricType == "counter") { + def schemas(): Map[String, MetricBuilder] = if (metricType == "counter") { Map( - metricName -> CounterSchema( + metricName -> MetricBuilder( name = metricName.split("\\/"), percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), + metricType = CounterType, statsReceiver = null - ))) + )) } else { Map( - metricName -> HistogramSchema( + metricName -> MetricBuilder( name = metricName.split("\\/"), percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), + metricType = HistogramType, statsReceiver = null - ))) + )) } } val request = Request("http://$HOST:$PORT/admin/metric_metadata.json?name=" + requestParam) diff --git a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala index 90ebfcf9..0aa1818b 100644 --- a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala @@ -2,6 +2,7 @@ package com.twitter.server.handler.exp import com.twitter.conversions.DurationOps._ import com.twitter.finagle.http.Request +import com.twitter.finagle.stats.MetricBuilder.{CounterType, GaugeType, HistogramType} import com.twitter.finagle.stats.exp.{ Expression, ExpressionSchema, @@ -9,15 +10,7 @@ import com.twitter.finagle.stats.exp.{ GreaterThan, MonotoneThresholds } -import com.twitter.finagle.stats.{ - CounterSchema, - GaugeSchema, - HistogramSchema, - InMemoryStatsReceiver, - MetricBuilder, - MetricSchema, - SchemaRegistry -} +import com.twitter.finagle.stats.{InMemoryStatsReceiver, MetricBuilder, SchemaRegistry} import com.twitter.server.handler.MetricExpressionHandler import com.twitter.server.util.{AdminJsonConverter, JsonUtils, MetricSchemaSource} import com.twitter.util.{Await, Awaitable, Duration} @@ -30,10 +23,11 @@ class MetricExpressionHandlerTest extends AnyFunSuite { val sr = new InMemoryStatsReceiver - val successMb = CounterSchema(MetricBuilder(name = Seq("success"), statsReceiver = sr)) + val successMb = MetricBuilder(name = Seq("success"), metricType = CounterType, statsReceiver = sr) val failuresMb = - CounterSchema(MetricBuilder(name = Seq("failures"), statsReceiver = sr)) - val latencyMb = HistogramSchema(MetricBuilder(name = Seq("latency"), statsReceiver = sr)) + MetricBuilder(name = Seq("failures"), metricType = CounterType, statsReceiver = sr) + val latencyMb = + MetricBuilder(name = Seq("latency"), metricType = HistogramType, statsReceiver = sr) val successRateExpression = ExpressionSchema( @@ -57,7 +51,7 @@ class MetricExpressionHandlerTest extends AnyFunSuite { val expressionRegistry = new SchemaRegistry { def hasLatchedCounters: Boolean = false - def schemas(): Map[String, MetricSchema] = Map.empty + def schemas(): Map[String, MetricBuilder] = Map.empty val expressions: _root_.scala.Predef.Map[ _root_.com.twitter.finagle.stats.exp.ExpressionSchemaKey, @@ -69,7 +63,7 @@ class MetricExpressionHandlerTest extends AnyFunSuite { val latchedRegistry = new SchemaRegistry { def hasLatchedCounters: Boolean = true - def schemas(): Map[String, MetricSchema] = Map.empty + def schemas(): Map[String, MetricBuilder] = Map.empty def expressions(): Map[ExpressionSchemaKey, ExpressionSchema] = expressionSchemaMap } @@ -206,7 +200,7 @@ class MetricExpressionHandlerTest extends AnyFunSuite { test("translate expressions - gauges") { val connMb = - GaugeSchema(MetricBuilder(name = Seq("client", "connections"), statsReceiver = sr)) + MetricBuilder(name = Seq("client", "connections"), metricType = GaugeType, statsReceiver = sr) val result = MetricExpressionHandler.translateToQuery(Expression(connMb)) assert(result == "client/connections") } diff --git a/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala b/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala index d7cb6232..627503e4 100644 --- a/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala +++ b/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala @@ -2,12 +2,13 @@ package com.twitter.server.util import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.scala.{DefaultScalaModule, ScalaObjectMapper} +import com.twitter.finagle.stats.MetricBuilder.{CounterType, GaugeType, HistogramType} import com.twitter.finagle.stats._ import org.scalatest.funsuite.AnyFunSuite class MetricSchemaJsonModuleTest extends AnyFunSuite { - private val counterSchema = CounterSchema( + private val counterSchema = MetricBuilder( keyIndicator = true, description = "Counts how many cools are seen", @@ -18,9 +19,10 @@ class MetricSchemaJsonModuleTest extends AnyFunSuite { name = Seq("my", "cool", "counter"), processPath = Some("dc/role/zone/service"), percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), + metricType = CounterType, statsReceiver = null - )) - private val gaugeSchema = GaugeSchema( + ) + private val gaugeSchema = MetricBuilder( keyIndicator = false, description = "Measures how fine the downstream system is", @@ -31,14 +33,16 @@ class MetricSchemaJsonModuleTest extends AnyFunSuite { name = Seq("your", "fine", "gauge"), processPath = Some("dc/your_role/zone/your_service"), percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), + metricType = GaugeType, statsReceiver = null - )) - private val histogramSchema = HistogramSchema( + ) + private val histogramSchema = MetricBuilder( name = Seq("my", "only", "histo"), percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), + metricType = HistogramType, statsReceiver = null - )) + ) private val topLevelFieldNameSet = Set( @@ -109,12 +113,12 @@ class MetricSchemaJsonModuleTest extends AnyFunSuite { } test("GaugeSchema does not include counterishGauge unless it is true") { - val counterish = gaugeSchema.copy(metricBuilder = gaugeSchema.metricBuilder.withCounterishGauge) + val counterish = gaugeSchema.withCounterishGauge for (schema <- Seq(gaugeSchema, counterish)) { val serializedString = AdminJsonConverter.writeToString(schema) val jsonMap = jsonStrToMap(serializedString) - val expected = if (schema.metricBuilder.isCounterishGauge) Some(true) else None + val expected = if (schema.isCounterishGauge) Some(true) else None assert(jsonMap.get("counterish_gauge") == expected) } } diff --git a/server/src/test/scala/com/twitter/server/util/MetricSchemaSourceTest.scala b/server/src/test/scala/com/twitter/server/util/MetricSchemaSourceTest.scala index 4a54e5f0..7b34ea96 100644 --- a/server/src/test/scala/com/twitter/server/util/MetricSchemaSourceTest.scala +++ b/server/src/test/scala/com/twitter/server/util/MetricSchemaSourceTest.scala @@ -1,13 +1,14 @@ package com.twitter.server.util +import com.twitter.finagle.stats.MetricBuilder.{CounterType, GaugeType, HistogramType} import com.twitter.finagle.stats._ import com.twitter.finagle.stats.exp.{ExpressionSchema, ExpressionSchemaKey} import org.scalatest.funsuite.AnyFunSuite class MetricSchemaSourceTest extends AnyFunSuite { - private val schemaMap: Map[String, MetricSchema] = Map( - "my/cool/counter" -> CounterSchema( + private val schemaMap: Map[String, MetricBuilder] = Map( + "my/cool/counter" -> MetricBuilder( keyIndicator = true, description = "Counts how many cools are seen", @@ -18,9 +19,10 @@ class MetricSchemaSourceTest extends AnyFunSuite { name = Seq("my", "cool", "counter"), processPath = Some("dc/role/zone/service"), percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), + metricType = CounterType, statsReceiver = null - )), - "your/fine/gauge" -> GaugeSchema( + ), + "your/fine/gauge" -> MetricBuilder( keyIndicator = false, description = "Measures how fine the downstream system is", @@ -31,25 +33,27 @@ class MetricSchemaSourceTest extends AnyFunSuite { name = Seq("your", "fine", "gauge"), processPath = Some("dc/your_role/zone/your_service"), percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), + metricType = GaugeType, statsReceiver = null - )), - "my/only/histo" -> HistogramSchema( + ), + "my/only/histo" -> MetricBuilder( name = Seq("my", "only", "histo"), percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), + metricType = HistogramType, statsReceiver = null - )) + ) ) private val latchedPopulatedRegistry: SchemaRegistry = new SchemaRegistry { def hasLatchedCounters: Boolean = true - def schemas(): Map[String, MetricSchema] = schemaMap + def schemas(): Map[String, MetricBuilder] = schemaMap def expressions(): Map[ExpressionSchemaKey, ExpressionSchema] = Map.empty } private val unlatchedPopulatedRegistry: SchemaRegistry = new SchemaRegistry { def hasLatchedCounters: Boolean = false - def schemas(): Map[String, MetricSchema] = schemaMap + def schemas(): Map[String, MetricBuilder] = schemaMap def expressions(): Map[ExpressionSchemaKey, ExpressionSchema] = Map.empty } From 3f553a47c408141e7f3002f779390826b682adbf Mon Sep 17 00:00:00 2001 From: Felix Bruckmeier Date: Fri, 4 Jun 2021 19:35:30 +0000 Subject: [PATCH 260/409] *: Upgrade sbt to 1.5.3 Problem sbt supports Scala 3 natively in the recent versions (e.g. 1.5.x) and I want the build to be ready for Scala 3. Solution Updating sbt from 1.3.10 to 1.5.3 Result sbt is updated to 1.5.3 and most warnings addressed (i.e. auto-reloading, unused settings, and migration to slash-syntax for settings) Signed-off-by: Moses Nakamura Differential Revision: https://phabricator.twitter.biz/D680505 --- build.sbt | 23 +++++++++++++---------- project/build.properties | 2 +- project/plugins.sbt | 1 - sbt | 18 ++++++++++-------- 4 files changed, 24 insertions(+), 20 deletions(-) diff --git a/build.sbt b/build.sbt index 74f73596..2cda8d66 100644 --- a/build.sbt +++ b/build.sbt @@ -1,5 +1,8 @@ import scoverage.ScoverageKeys +Global / onChangedBuildSource := ReloadOnSourceChanges +Global / excludeLintKeys += scalacOptions + // All Twitter library releases are date versioned as YY.MM.patch val releaseVersion = "21.6.0-SNAPSHOT" @@ -16,7 +19,7 @@ def util(which: String) = "com.twitter" %% ("util-" + which) % releaseVersion def finagle(which: String) = "com.twitter" %% ("finagle-" + which) % releaseVersion lazy val noPublishSettings = Seq( - skip in publish := true + publish / skip := true ) def gcJavaOptions: Seq[String] = { @@ -78,7 +81,7 @@ lazy val sharedSettings = Seq( organization := "com.twitter", scalaVersion := "2.12.12", crossScalaVersions := Seq("2.12.12", "2.13.1"), - fork in Test := true, // We have to fork to get the JavaOptions + Test / fork := true, // We have to fork to get the JavaOptions libraryDependencies ++= Seq( // See https://www.scala-sbt.org/0.13/docs/Testing.html#JUnit "com.novocode" % "junit-interface" % "0.11" % "test", @@ -106,20 +109,20 @@ lazy val sharedSettings = Seq( "utf8" ), javacOptions ++= Seq("-Xlint:unchecked", "-source", "1.8", "-target", "1.8"), - javacOptions in doc := Seq("-source", "1.8"), + doc / javacOptions := Seq("-source", "1.8"), javaOptions ++= Seq( "-Djava.net.preferIPv4Stack=true", "-XX:+AggressiveOpts", "-server" ), javaOptions ++= gcJavaOptions, - javaOptions in Test ++= travisTestJavaOptions, + Test / javaOptions ++= travisTestJavaOptions, // This is bad news for things like com.twitter.util.Time - parallelExecution in Test := false, + Test / parallelExecution := false, // -a: print stack traces for failing asserts testOptions += Tests.Argument(TestFrameworks.JUnit, "-a"), // Sonatype publishing - publishArtifact in Test := false, + Test / publishArtifact := false, pomIncludeRepository := { _ => false }, publishMavenStyle := true, publishConfiguration := publishConfiguration.value.withOverwrite(true), @@ -260,15 +263,15 @@ lazy val twitterServerDoc = (project in file("doc")) name := "twitter-server-doc", sharedSettings, Seq( - scalacOptions in doc ++= Seq("-doc-title", "TwitterServer", "-doc-version", version.value), - includeFilter in Sphinx := ("*.html" | "*.png" | "*.js" | "*.css" | "*.gif" | "*.txt") + doc / scalacOptions ++= Seq("-doc-title", "TwitterServer", "-doc-version", version.value), + Sphinx / includeFilter := ("*.html" | "*.png" | "*.js" | "*.css" | "*.gif" | "*.txt") ) ) .configs(DocTest).settings(inConfig(DocTest)(Defaults.testSettings): _*) .settings( - unmanagedSourceDirectories in DocTest += baseDirectory.value / "src/sphinx/code", + DocTest / unmanagedSourceDirectories += baseDirectory.value / "src/sphinx/code", // Make the "test" command run both, test and doctest:test - test := Seq(test in Test, test in DocTest).dependOn.value + test := Seq(Test / test, DocTest / test).dependOn.value ) .dependsOn(twitterServer) diff --git a/project/build.properties b/project/build.properties index 797e7ccf..67d27a1d 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.3.10 +sbt.version=1.5.3 diff --git a/project/plugins.sbt b/project/plugins.sbt index b095855a..08dab906 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -4,6 +4,5 @@ resolvers ++= Seq( ) addSbtPlugin("com.typesafe.sbt" % "sbt-site" % "1.3.0") -addSbtPlugin("io.get-coursier" % "sbt-coursier" % "1.0.0-RC13") addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.6.1") addSbtPlugin("com.eed3si9n" % "sbt-unidoc" % "0.4.1") diff --git a/sbt b/sbt index 4cb7d05e..2441ede7 100755 --- a/sbt +++ b/sbt @@ -1,21 +1,23 @@ #!/bin/bash -sbtver=1.3.10 -sbtjar=sbt-launch.jar -sbtsha128=f49861befe36746728182ea7efb30096c724b0d7 +set -eo pipefail + +sbtver="1.5.3" +sbtjar="sbt-launch-$sbtver.jar" +sbtsha128="1ca2d0ee419a1f82f512f2aa8556d6e262b37961" sbtrepo="https://repo1.maven.org/maven2/org/scala-sbt/sbt-launch" -if [ ! -f $sbtjar ]; then +if [ ! -f "sbt-launch.jar" ]; then echo "downloading $PWD/$sbtjar" 1>&2 - if ! curl --location --silent --fail --remote-name $sbtrepo/$sbtver/$sbtjar; then + if ! curl --location --silent --fail -o "sbt-launch.jar" "$sbtrepo/$sbtver/$sbtjar"; then exit 1 fi fi -checksum=`openssl dgst -sha1 $sbtjar | awk '{ print $2 }'` +checksum=`openssl dgst -sha1 sbt-launch.jar | awk '{ print $2 }'` if [ "$checksum" != $sbtsha128 ]; then - echo "bad $PWD/$sbtjar. delete $PWD/$sbtjar and run $0 again." + echo "bad $PWD/sbt-launch.jar. delete $PWD/sbt-launch.jar and run $0 again." exit 1 fi @@ -24,4 +26,4 @@ fi java -ea \ $SBT_OPTS \ $JAVA_OPTS \ - -jar $sbtjar "$@" + -jar "sbt-launch.jar" "$@" From 4adf394308025ec1c0c326ca36d1927e01b6b6c1 Mon Sep 17 00:00:00 2001 From: Flavio Brasil Date: Sun, 6 Jun 2021 01:55:19 +0000 Subject: [PATCH 261/409] twitter-server: lock-free metric store # Problem The implementation of `MetricStore` uses synchronized blocks to handle the refresh of metrics. I noticed that it becomes a significant contention point for Strato. JFR profiling session: {F32332182} # Solution Optimize the implementation to use a lock-free mechanism. # Notes There's a minor behavior change that I believe isn't a problem: the previous implementation would block reads while the map was being refreshed but the new lock-free version allows other threads to read the stale map while the refresh is happening. Differential Revision: https://phabricator.twitter.biz/D682705 --- .../twitter/server/util/MetricSource.scala | 26 ++++++----- .../server/util/MetricSourceTest.scala | 45 +++++++++++++++++-- 2 files changed, 58 insertions(+), 13 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/util/MetricSource.scala b/server/src/main/scala/com/twitter/server/util/MetricSource.scala index 8f5ff191..79e0b276 100644 --- a/server/src/main/scala/com/twitter/server/util/MetricSource.scala +++ b/server/src/main/scala/com/twitter/server/util/MetricSource.scala @@ -5,6 +5,7 @@ import com.twitter.finagle.stats.{StatEntry, StatsRegistry} import com.twitter.finagle.util.LoadService import com.twitter.server.util.MetricSource.MetricTypeInfo import com.twitter.util.{Duration, Time} +import java.util.concurrent.atomic.AtomicReference private[server] object MetricSource { lazy val registry: Seq[StatsRegistry] = LoadService[StatsRegistry]() @@ -19,14 +20,19 @@ private[server] object MetricSource { private[server] class MetricSource( registry: () => Seq[StatsRegistry] = { () => MetricSource.registry }, refreshInterval: Duration = 1.second) { - private[this] var lastRefresh = Time.now - refreshInterval - private[this] var underlying: Map[String, StatEntry] = Map.empty + + // null while the map is being refreshed + private[this] val lastRefresh = new AtomicReference(Time.now - refreshInterval) + + @volatile private[this] var underlying: Map[String, StatEntry] = Map.empty private[this] def refresh(): Unit = { - if (Time.now - lastRefresh > refreshInterval) { + val last = lastRefresh.get() + if (last != null && Time.now - last > refreshInterval && + lastRefresh.compareAndSet(last, null)) { val newStats = registry().foldLeft(Map[String, StatEntry]()) { (map, r) => map ++ r() } underlying = newStats - lastRefresh = Time.now + lastRefresh.set(Time.now) } } @@ -34,7 +40,7 @@ private[server] class MetricSource( * @note this relies on the fact that there is only one StatsRegistry and that it is * the finagle implementation. */ - def hasLatchedCounters(): Boolean = synchronized { + def hasLatchedCounters(): Boolean = { val statsRegistry = registry().headOption.getOrElse { throw new RuntimeException("No StatsRegistries available") } @@ -42,32 +48,32 @@ private[server] class MetricSource( } /** Returns the entry for `key` if it exists */ - def get(key: String): Option[MetricSource.Entry] = synchronized { + def get(key: String): Option[MetricSource.Entry] = { refresh() for (s <- underlying.get(key)) yield MetricSource.Entry(key, s.delta, s.value) } /** Returns the entry for `key` with type info if the entry exists */ - def getType(key: String): Option[MetricSource.MetricTypeInfo] = synchronized { + def getType(key: String): Option[MetricSource.MetricTypeInfo] = { refresh() for (s <- underlying.get(key)) yield MetricSource.MetricTypeInfo(key, s.metricType) } /** Returns true if the map contains `key` and false otherwise. */ - def contains(key: String): Boolean = synchronized { + def contains(key: String): Boolean = { refresh() underlying.contains(key) } /** Returns the set of stat keys. */ - def keySet: Set[String] = synchronized { + def keySet: Set[String] = { refresh() underlying.keySet } /** Returns a map of metric names to their types */ - def typeMap: Iterable[MetricTypeInfo] = synchronized { + def typeMap: Iterable[MetricTypeInfo] = { refresh() underlying.map { case (name, statEntry) => MetricSource.MetricTypeInfo(name, statEntry.metricType) diff --git a/server/src/test/scala/com/twitter/server/util/MetricSourceTest.scala b/server/src/test/scala/com/twitter/server/util/MetricSourceTest.scala index 707ddec9..3a1ee275 100644 --- a/server/src/test/scala/com/twitter/server/util/MetricSourceTest.scala +++ b/server/src/test/scala/com/twitter/server/util/MetricSourceTest.scala @@ -1,19 +1,24 @@ package com.twitter.server.util import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.stats.{StatsRegistry, StatEntry} -import com.twitter.util.Time +import com.twitter.finagle.stats.{StatEntry, StatsRegistry} +import com.twitter.util.{Await, Future, FuturePool, Time} +import java.util.concurrent.{CountDownLatch, Executors} import org.scalatest.funsuite.AnyFunSuite private[server] object MetricSourceTest { class Ctx { case class Entry(delta: Double, value: Double, metricType: String) extends StatEntry private[twitter] var underlying = Map[String, StatEntry]() + private[twitter] var refreshes = 0 val sr = new StatsRegistry { def apply() = underlying; override val latched: Boolean = false } - val registry = { () => Seq(sr) } + val registry = { () => + refreshes += 1 + Seq(sr) + } val source = new MetricSource(registry, 1.second) } } @@ -32,6 +37,7 @@ class MetricSourceTest extends AnyFunSuite { tc.advance(1.second) assert(source.get("clnt/foo/requests").get.delta == 0.0) assert(source.get("clnt/foo/requests").get.value == 10.0) + assert(refreshes == 1) } } @@ -45,6 +51,7 @@ class MetricSourceTest extends AnyFunSuite { tc.advance(1.second) assert(source.contains("clnt/foo/requests") == true) + assert(refreshes == 1) } } @@ -59,6 +66,38 @@ class MetricSourceTest extends AnyFunSuite { assert(source.keySet == Set.empty[String]) tc.advance(1.second) assert(source.keySet == Set("clnt/foo/requests", "clnt/foo/success")) + assert(refreshes == 1) + } + } + + test("only one thread refreshes") { + Time.withCurrentTimeFrozen { tc => + val ctx = new Ctx + import ctx._ + val exec = Executors.newCachedThreadPool() + val pool = FuturePool(exec) + + try { + underlying = Map("clnt/foo/requests" -> Entry(0.0, 10.0, "counter")) + assert(source.get("clnt/foo/requests") == None) + + tc.advance(1.second) + + val cdl = new CountDownLatch(1) + val fut = + Future.collect(List.fill(20)(pool { + cdl.await() + source.get("clnt/foo/requests") + })) + cdl.countDown() + Await.result(fut, 1.second) + + assert(source.get("clnt/foo/requests").get.delta == 0.0) + assert(source.get("clnt/foo/requests").get.value == 10.0) + assert(refreshes == 1) + } finally { + exec.shutdown() + } } } } From 92527ef563acbe5fb04f46fdab99adb5c42f6daf Mon Sep 17 00:00:00 2001 From: Felix Bruckmeier Date: Tue, 8 Jun 2021 18:48:12 +0000 Subject: [PATCH 262/409] *: Update Scala 2.13.1 to 2.13.6 Problem The latest Scala 2.13.x version is compatible with Scala 3.0.0 and includes many useful compiler-tools (e.g. warnings) to make the transition - and development in general - easier. Even if we don't have a real Scala 3 cross-build, 2.13.6 can be consumed by Scala 3 projects. Solution Updating Scala 2.13.1 to 2.13.6 Result The build uses Scala 2.13.6. I also used the 2.13.x version as default as it has many extremely useful warnings that 2.12.x doesn't have. The only part that needed fixing is some reflection code as 2.13.5 changed encoding of method-parameter names. Signed-off-by: Moses Nakamura Differential Revision: https://phabricator.twitter.biz/D680820 --- build.sbt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index 2cda8d66..fb2cd7e0 100644 --- a/build.sbt +++ b/build.sbt @@ -79,8 +79,8 @@ def travisTestJavaOptions: Seq[String] = { lazy val sharedSettings = Seq( version := releaseVersion, organization := "com.twitter", - scalaVersion := "2.12.12", - crossScalaVersions := Seq("2.12.12", "2.13.1"), + scalaVersion := "2.13.6", + crossScalaVersions := Seq("2.12.12", "2.13.6"), Test / fork := true, // We have to fork to get the JavaOptions libraryDependencies ++= Seq( // See https://www.scala-sbt.org/0.13/docs/Testing.html#JUnit From 26fa974af64eab07d13605db87983b71a2421073 Mon Sep 17 00:00:00 2001 From: Dorothy Ordogh Date: Wed, 9 Jun 2021 00:27:44 +0000 Subject: [PATCH 263/409] finagle|finatra|scrooge|ts|util: Use sbt version in GH actions cache key Problem The builds were failing because pre-sbt upgrade, the existing sbt launch jar was cached, which caused a conflict when trying to use the upgraded sbt launch jar (which had the same name as the old one). Solution Add the sbt version to the cache key so that when the sbt version is upgraded, the cache will be invalidated and everything will run smoothly. JIRA Issues: CSL-11029 Differential Revision: https://phabricator.twitter.biz/D683985 --- .github/workflows/continuous-integration.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index acb89355..5310ef3f 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -3,6 +3,7 @@ name: continuous integration env: JAVA_OPTS: "-Dsbt.log.noformat=true" TRAVIS: "true" # pretend we're TravisCI + SBT_VERSION: 1.5.3 defaults: run: @@ -41,7 +42,7 @@ jobs: ~/.ivy2/cache ~/.ivy2/local/com.twitter ~/.sbt - key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ matrix.scala }}-${{ matrix.java }} + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ env.SBT_VERSION }}-${{ matrix.scala }}-${{ matrix.java }} - name: update cache run: | if [ -f ~/.ivy2/cache ]; then From d8e915d9b6ab3c99ff4cd29d84cd939d8c19b1c5 Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Wed, 9 Jun 2021 19:02:47 +0000 Subject: [PATCH 264/409] twitter-server: Sort sublinks alphabetically Problem/Solution Sort TwitterServer Admin sublinks alphabetically to better reason about and maintain them. JIRA Issues: CSL-11025 Differential Revision: https://phabricator.twitter.biz/D684429 --- CHANGELOG.rst | 6 ++++++ .../src/main/scala/com/twitter/server/AdminHttpServer.scala | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9cd59d36..9d40803f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,12 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Runtime Behavior Changes +~~~~~~~~~~~~~~~~~~~~~~~~ + +* Sort sublinks from Admin UI alphabetically. ``PHAB_ID=D684429`` + + 21.5.0 ------ diff --git a/server/src/main/scala/com/twitter/server/AdminHttpServer.scala b/server/src/main/scala/com/twitter/server/AdminHttpServer.scala index 3b5b9a60..92cda850 100644 --- a/server/src/main/scala/com/twitter/server/AdminHttpServer.scala +++ b/server/src/main/scala/com/twitter/server/AdminHttpServer.scala @@ -272,7 +272,7 @@ trait AdminHttpServer { self: App with Stats => .groupBy(_.group) .flatMap { case (groupOpt, rts) => - val links = rts.map(routeToIndexLink) + val links = rts.map(routeToIndexLink).sorted(IndexView.EntryOrdering) groupOpt match { case Some(group) => Seq(IndexView.Group(group, links)) case None => links From d3eb9be60a77b7d816560ed478bf5e98ddef0784 Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Tue, 15 Jun 2021 21:12:32 +0000 Subject: [PATCH 265/409] twitter-server: Add more information on tracing page Problem/Solution: Add more information on how to enable/disable tracing from the Admin tracing page. JIRA Issues: CSL-11019 Differential Revision: https://phabricator.twitter.biz/D687337 --- CHANGELOG.rst | 2 ++ .../server/handler/TracingHandler.scala | 25 ++++++++++++------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9d40803f..1813d35f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -12,6 +12,8 @@ Runtime Behavior Changes * Sort sublinks from Admin UI alphabetically. ``PHAB_ID=D684429`` +* Added more information on how to enable/disable tracing in admin/tracing UI. + ``PHAB_ID=D687337`` 21.5.0 ------ diff --git a/server/src/main/scala/com/twitter/server/handler/TracingHandler.scala b/server/src/main/scala/com/twitter/server/handler/TracingHandler.scala index 09725aa8..11e6ea11 100644 --- a/server/src/main/scala/com/twitter/server/handler/TracingHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/TracingHandler.scala @@ -54,15 +54,22 @@ class TracingHandler extends Service[Request, Response] { } val tracing = FinagleTracing.instance.get - val msg = if (params.getOrElse("enable", "") == "true") { - tracing.enable() - "Enabled Finagle tracing" - } else if (params.getOrElse("disable", "") == "true") { - tracing.disable() - "Disabling Finagle tracing" - } else { - "Could not figure out what you wanted to do with tracing. " + - "Either enable or disable it. This is what we got: " + params + + val msg = (params.get("enable"), params.get("disable")) match { + case (Some("true"), Some("true")) => + """You set tracing to be enabled and disabled at the same time. + |Either enable (/admin/tracing?enable=true) or disable (/admin/tracing?disable=true) it. + |""".stripMargin + case (Some("true"), _) => + tracing.enable() + "Enabled Finagle tracing" + case (_, Some("true")) => + tracing.disable() + "Disabling Finagle tracing" + case _ => + """You did not set the parameter for tracing. + |Either enable (/admin/tracing?enable=true) or disable (/admin/tracing?disable=true) it. + |""".stripMargin } log.info(msg) From e7b47f37e7078462fc9683266c327c2e58e7d5ed Mon Sep 17 00:00:00 2001 From: Lily He Date: Wed, 16 Jun 2021 23:32:10 +0000 Subject: [PATCH 266/409] twitter-server: responsive UI for admin stack module registry Problem: In /admin/clients and /admin/servers, when the text in the parameter table of the Stack Module page gets too long, the entire table shifts to the bottom of the page. This gives the false impression that the description of the module is missing. Solution: Contain the panel in bootstrap grid columns. Break the long text so it wraps around to the next line. Result: The table will always display on the right no matter the width of the screen. The table width will grow and shrink depending on the screen width. JIRA Issues: CSL-11037 Differential Revision: https://phabricator.twitter.biz/D686836 --- CHANGELOG.rst | 3 +++ server/src/main/resources/twitter-server/css/index.css | 10 ++++++++++ .../com/twitter/server/view/StackRegistryView.scala | 8 ++++---- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1813d35f..a401299d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -15,6 +15,9 @@ Runtime Behavior Changes * Added more information on how to enable/disable tracing in admin/tracing UI. ``PHAB_ID=D687337`` +* Resize stack module description table based on the width of the window in + Downstream Clients and Listening Servers pages in Admin UI. ``PHAB_ID=D686836`` + 21.5.0 ------ diff --git a/server/src/main/resources/twitter-server/css/index.css b/server/src/main/resources/twitter-server/css/index.css index 62f12101..6c92c601 100644 --- a/server/src/main/resources/twitter-server/css/index.css +++ b/server/src/main/resources/twitter-server/css/index.css @@ -150,6 +150,16 @@ nav .subnav-title { cursor: pointer; } +/** Client Stack Registry and Server Stack Registry */ +.stack-value { + word-break: break-all; +} + +.stack-module-panel { + display:inline-block; + min-width:70%; +} + /** Shared */ pre { diff --git a/server/src/main/scala/com/twitter/server/view/StackRegistryView.scala b/server/src/main/scala/com/twitter/server/view/StackRegistryView.scala index f0689b3f..35b541f6 100644 --- a/server/src/main/scala/com/twitter/server/view/StackRegistryView.scala +++ b/server/src/main/scala/com/twitter/server/view/StackRegistryView.scala @@ -18,7 +18,7 @@ private[server] object StackRegistryView { (for ((field, value) <- params) yield { s""" ${escapeHtml(field)} - ${escapeHtml(value)} + ${escapeHtml(value)} """ }).mkString("\n") @@ -45,7 +45,7 @@ private[server] object StackRegistryView {
    -
    +
    • Stack Modules
      • ${(for (StackRegistry.Module(role, _, _) <- modules) yield { @@ -54,10 +54,10 @@ private[server] object StackRegistryView {
    -
    +
    ${(for (StackRegistry.Module(role, desc, params) <- modules) yield { s"""
    -
    +
    ${escapeHtml(role)}

    ${escapeHtml(desc)}

    From d073137e13ac3719c48cf1d1c2d00b1c5aec1a9c Mon Sep 17 00:00:00 2001 From: Ruben Oanta Date: Mon, 21 Jun 2021 18:10:50 +0000 Subject: [PATCH 267/409] csl-libs: bump github workflows scala version Problem / Solution We recently bumped our scala version to 2.13.6 but neglected to update it for our github actions CI integration. This patch addresses this for the rest of our libs. Differential Revision: https://phabricator.twitter.biz/D691506 --- .github/workflows/continuous-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 5310ef3f..a0474700 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -20,7 +20,7 @@ jobs: strategy: fail-fast: false matrix: - scala: [2.12.12, 2.13.1] + scala: [2.12.12, 2.13.6] java: ['1.8', '1.11'] runs-on: ubuntu-latest steps: From 487511a575dedb5e0b5f13b49fecc142def64b55 Mon Sep 17 00:00:00 2001 From: Ruben Oanta Date: Tue, 22 Jun 2021 15:25:13 +0000 Subject: [PATCH 268/409] Twitter-oss: Prepare OSS libraries for release 21.6.0 Problem We want to release the next versions of our Twitter OSS libraries 21.6.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D692183 --- CHANGELOG.rst | 3 +++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a401299d..8e1fda2d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +21.6.0 +------ + Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/README.md b/README.md index 90042271..d4a12216 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index fb2cd7e0..e79e2366 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.6.0-SNAPSHOT" +val releaseVersion = "21.6.0" val jacksonVersion = "2.11.2" val jacksonLibs = Seq( From 9e8d1cad03008eeab72283320fceda2bd3a885d8 Mon Sep 17 00:00:00 2001 From: Ruben Oanta Date: Tue, 22 Jun 2021 15:51:56 +0000 Subject: [PATCH 269/409] Twitter-oss: update OSS libraries post-release to 21.7.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 21.7.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D692206 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d4a12216..90042271 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index e79e2366..d685edec 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.6.0" +val releaseVersion = "21.7.0-SNAPSHOT" val jacksonVersion = "2.11.2" val jacksonLibs = Seq( From 18ac702262e6c302f798f8e32d7a9d63f3961eda Mon Sep 17 00:00:00 2001 From: Lily He Date: Tue, 22 Jun 2021 21:58:20 +0000 Subject: [PATCH 270/409] twitter-oss: Move templates to PULL_REQUEST_TEMPLATE folder Problem: GitHub needs the special keyword "closes" to automatically close PRs when a commit is merged. Solution: Add "closes twitter/$REPOSITORY#PR_NUMBER" to the pull request template so PRs from open source automatically close when the commit is merged in. This avoids manually closing the PR when possible. Also, move all the pull request templates to a PULL_REQUEST_TEMPLATE folder because this is the recommended standard from GitHub Differential Revision: https://phabricator.twitter.biz/D692645 --- .../pull_request_template.md} | 2 ++ 1 file changed, 2 insertions(+) rename .github/{PULL_REQUEST_TEMPLATE.md => PULL_REQUEST_TEMPLATE/pull_request_template.md} (91%) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md similarity index 91% rename from .github/PULL_REQUEST_TEMPLATE.md rename to .github/PULL_REQUEST_TEMPLATE/pull_request_template.md index bfb079c9..72aa251d 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md @@ -13,3 +13,5 @@ Result What will change as a result of your pull request? Note that sometimes this section is unnecessary because it is self-explanatory based on the solution. + +Closes twitter/twitter-server#PR_NUMBER From d8a6f510b1bb73087d119fb2eab2dcd24d128c28 Mon Sep 17 00:00:00 2001 From: Lily He Date: Thu, 24 Jun 2021 14:34:34 +0000 Subject: [PATCH 271/409] twitter-oss: delete old issue templates and use Twitter org ones Problem: The old version of issue templates is a ISSUE_TEMPLATE.md file. GitHub recommends updating to the new issue template workflow, where templates are placed in a subdirectory called ISSUE_TEMPLATE. The Twitter organization on GitHub already provides templates for bug report, feature request, and custom issues in this new workflow. When someone creates a GitHub issue in Twitter repositories, these are the templates to choose from. Solution: Delete ISSUE_TEMPLATE.md from finagle, finatra, scrooge, twitter-server, util and dodo. This template is not used anyways because the Twitter organization templates are applied. JIRA Issues: CSL-8687 Differential Revision: https://phabricator.twitter.biz/D692804 --- .github/ISSUE_TEMPLATE.md | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE.md diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md deleted file mode 100644 index ef3b9bc7..00000000 --- a/.github/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,13 +0,0 @@ -One line summary of the issue here. - -### Expected behavior - -As concisely as possible, describe the expected behavior. - -### Actual behavior - -As concisely as possible, describe the observed behavior. - -### Steps to reproduce the behavior - -Please list all relevant steps to reproduce the observed behavior. \ No newline at end of file From 2bd4d8b53fa6bdeb3646d9aae89fd4fa3bba9194 Mon Sep 17 00:00:00 2001 From: Felix Bruckmeier Date: Wed, 30 Jun 2021 21:22:09 +0000 Subject: [PATCH 272/409] util|finagle|finatra|twitter-server|scrooge: Bump Scalacheck to 1.15.4 in sbt Problem Scala 3 requires Scalacheck 1.15.x but the sbt builds currently uses 1.14.x Solution Upgrade Scalacheck in build.sbt to 1.15.4 to simplify Scala 3 cross-building. Result The tests use Scalacheck 1.15.4. We still use `scalatestplus scalacheck-1-14` because there is no `1-15` version available for Scalatest 3.1.x. Closes twitter/util#291 Signed-off-by: Lily He JIRA Issues: CSL-11064 Differential Revision: https://phabricator.twitter.biz/D691691 --- CHANGELOG.rst | 4 ++++ build.sbt | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8e1fda2d..b90f47a7 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,10 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Runtime Behavior Changes +~~~~~~~~~~~~~~~~~~~~~~~~ +* Update Scalatest to version 1.15.4 ``PHAB_ID=D691691`` + 21.6.0 ------ diff --git a/build.sbt b/build.sbt index d685edec..c46b0bc4 100644 --- a/build.sbt +++ b/build.sbt @@ -86,7 +86,7 @@ lazy val sharedSettings = Seq( // See https://www.scala-sbt.org/0.13/docs/Testing.html#JUnit "com.novocode" % "junit-interface" % "0.11" % "test", "org.mockito" % "mockito-all" % "1.9.5" % "test", - "org.scalacheck" %% "scalacheck" % "1.14.3" % "test", + "org.scalacheck" %% "scalacheck" % "1.15.4" % "test", "org.scalatest" %% "scalatest" % "3.1.2" % "test", "org.scalatestplus" %% "junit-4-12" % "3.1.2.0" % "test", "org.scalatestplus" %% "mockito-1-10" % "3.1.0.0" % "test", From 2d9e65dad7c7f82b719597fbff2b89b5e252e9fe Mon Sep 17 00:00:00 2001 From: Yi Cheng Date: Wed, 7 Jul 2021 00:58:04 +0000 Subject: [PATCH 273/409] [finagle] tag bazel-compatible Differential Revision: https://phabricator.twitter.biz/D699435 --- server/src/main/resources/BUILD | 1 + server/src/main/scala/BUILD | 1 + 2 files changed, 2 insertions(+) diff --git a/server/src/main/resources/BUILD b/server/src/main/resources/BUILD index 45a90181..07015f79 100644 --- a/server/src/main/resources/BUILD +++ b/server/src/main/resources/BUILD @@ -4,4 +4,5 @@ resources( "!BUILD*", "**/*", ], + tags = ["bazel-compatible"], ) diff --git a/server/src/main/scala/BUILD b/server/src/main/scala/BUILD index 70e2cb8c..200d2614 100644 --- a/server/src/main/scala/BUILD +++ b/server/src/main/scala/BUILD @@ -7,6 +7,7 @@ scala_library( repo = artifactory, ), scope = "forced", # Exposes a service-loaded class. + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/fasterxml/jackson/core:jackson-annotations", "3rdparty/jvm/com/fasterxml/jackson/core:jackson-core", From 7e8dc6675bd96962f2b2cf7480e545c3481d15d5 Mon Sep 17 00:00:00 2001 From: Yi Cheng Date: Wed, 7 Jul 2021 02:07:46 +0000 Subject: [PATCH 274/409] [finatra] tag bazel compatible Differential Revision: https://phabricator.twitter.biz/D700012 --- logback-classic/src/main/scala/BUILD | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/logback-classic/src/main/scala/BUILD b/logback-classic/src/main/scala/BUILD index 586236dc..1c273173 100644 --- a/logback-classic/src/main/scala/BUILD +++ b/logback-classic/src/main/scala/BUILD @@ -7,7 +7,10 @@ scala_library( repo = artifactory, ), scope = "forced", # Exposes a service-loaded class. - tags = ["logging_impl_check_whitelisted_target"], + tags = [ + "bazel-compatible", + "logging_impl_check_whitelisted_target", + ], dependencies = [ "3rdparty/jvm/ch/qos/logback:logback-classic", "3rdparty/jvm/ch/qos/logback:logback-core", From c7ba2e6b1654383f53a282f5d546ce1aa94b0df2 Mon Sep 17 00:00:00 2001 From: Yi Cheng Date: Wed, 7 Jul 2021 22:08:01 +0000 Subject: [PATCH 275/409] [twitter-server] tag bazel compatible Differential Revision: https://phabricator.twitter.biz/D700027 --- logback-classic/BUILD | 1 + logback-classic/src/main/resources/BUILD | 1 + logback-classic/src/test/scala/BUILD | 1 + .../src/main/scala/com/twitter/server/opencensus/BUILD | 1 + server/src/test/resources/BUILD | 1 + slf4j-jdk14/BUILD | 1 + slf4j-jdk14/src/main/resources/BUILD | 1 + slf4j-jdk14/src/main/scala/com/twitter/server/logging/BUILD | 1 + .../src/main/scala/com/twitter/server/slf4j/jdk14/BUILD | 1 + slf4j-log4j12/src/main/resources/BUILD | 1 + slf4j-log4j12/src/main/scala/BUILD | 5 ++++- slf4j-log4j12/src/test/scala/BUILD | 1 + 12 files changed, 15 insertions(+), 1 deletion(-) diff --git a/logback-classic/BUILD b/logback-classic/BUILD index 7cc8414b..b881bca3 100644 --- a/logback-classic/BUILD +++ b/logback-classic/BUILD @@ -1,4 +1,5 @@ target( + tags = ["bazel-compatible"], dependencies = [ "twitter-server/logback-classic/src/main/scala", ], diff --git a/logback-classic/src/main/resources/BUILD b/logback-classic/src/main/resources/BUILD index 45a90181..07015f79 100644 --- a/logback-classic/src/main/resources/BUILD +++ b/logback-classic/src/main/resources/BUILD @@ -4,4 +4,5 @@ resources( "!BUILD*", "**/*", ], + tags = ["bazel-compatible"], ) diff --git a/logback-classic/src/test/scala/BUILD b/logback-classic/src/test/scala/BUILD index e54c1ff5..638ae174 100644 --- a/logback-classic/src/test/scala/BUILD +++ b/logback-classic/src/test/scala/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/ch/qos/logback:logback-classic", "3rdparty/jvm/junit", diff --git a/opencensus/src/main/scala/com/twitter/server/opencensus/BUILD b/opencensus/src/main/scala/com/twitter/server/opencensus/BUILD index 7f118720..929d1311 100644 --- a/opencensus/src/main/scala/com/twitter/server/opencensus/BUILD +++ b/opencensus/src/main/scala/com/twitter/server/opencensus/BUILD @@ -1,6 +1,7 @@ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/io/opencensus:opencensus-api", "3rdparty/jvm/io/opencensus:opencensus-contrib-zpages", diff --git a/server/src/test/resources/BUILD b/server/src/test/resources/BUILD index 45a90181..07015f79 100644 --- a/server/src/test/resources/BUILD +++ b/server/src/test/resources/BUILD @@ -4,4 +4,5 @@ resources( "!BUILD*", "**/*", ], + tags = ["bazel-compatible"], ) diff --git a/slf4j-jdk14/BUILD b/slf4j-jdk14/BUILD index c8baf98e..2d853137 100644 --- a/slf4j-jdk14/BUILD +++ b/slf4j-jdk14/BUILD @@ -1,4 +1,5 @@ target( + tags = ["bazel-compatible"], dependencies = [ "twitter-server/slf4j-jdk14/src/main/scala", ], diff --git a/slf4j-jdk14/src/main/resources/BUILD b/slf4j-jdk14/src/main/resources/BUILD index 45a90181..07015f79 100644 --- a/slf4j-jdk14/src/main/resources/BUILD +++ b/slf4j-jdk14/src/main/resources/BUILD @@ -4,4 +4,5 @@ resources( "!BUILD*", "**/*", ], + tags = ["bazel-compatible"], ) diff --git a/slf4j-jdk14/src/main/scala/com/twitter/server/logging/BUILD b/slf4j-jdk14/src/main/scala/com/twitter/server/logging/BUILD index 0a75a117..aa6d6fdb 100644 --- a/slf4j-jdk14/src/main/scala/com/twitter/server/logging/BUILD +++ b/slf4j-jdk14/src/main/scala/com/twitter/server/logging/BUILD @@ -7,6 +7,7 @@ scala_library( repo = artifactory, ), strict_deps = True, + tags = ["bazel-compatible"], dependencies = [ "finagle/finagle-core", "util/util-app/src/main/scala", diff --git a/slf4j-jdk14/src/main/scala/com/twitter/server/slf4j/jdk14/BUILD b/slf4j-jdk14/src/main/scala/com/twitter/server/slf4j/jdk14/BUILD index d71dccf4..82ccc357 100644 --- a/slf4j-jdk14/src/main/scala/com/twitter/server/slf4j/jdk14/BUILD +++ b/slf4j-jdk14/src/main/scala/com/twitter/server/slf4j/jdk14/BUILD @@ -7,6 +7,7 @@ scala_library( repo = artifactory, ), strict_deps = True, + tags = ["bazel-compatible"], dependencies = [ "twitter-server/server/src/main/scala", "twitter-server/slf4j-jdk14/src/main/scala/com/twitter/server/logging", diff --git a/slf4j-log4j12/src/main/resources/BUILD b/slf4j-log4j12/src/main/resources/BUILD index 45a90181..07015f79 100644 --- a/slf4j-log4j12/src/main/resources/BUILD +++ b/slf4j-log4j12/src/main/resources/BUILD @@ -4,4 +4,5 @@ resources( "!BUILD*", "**/*", ], + tags = ["bazel-compatible"], ) diff --git a/slf4j-log4j12/src/main/scala/BUILD b/slf4j-log4j12/src/main/scala/BUILD index 778bc32f..8a344a78 100644 --- a/slf4j-log4j12/src/main/scala/BUILD +++ b/slf4j-log4j12/src/main/scala/BUILD @@ -7,7 +7,10 @@ scala_library( repo = artifactory, ), scope = "forced", # Exposes a service-loaded class. - tags = ["logging_impl_check_whitelisted_target"], + tags = [ + "bazel-compatible", + "logging_impl_check_whitelisted_target", + ], dependencies = [ "3rdparty/jvm/log4j", "3rdparty/jvm/org/slf4j:jul-to-slf4j", diff --git a/slf4j-log4j12/src/test/scala/BUILD b/slf4j-log4j12/src/test/scala/BUILD index 6ce980ff..af2d3975 100644 --- a/slf4j-log4j12/src/test/scala/BUILD +++ b/slf4j-log4j12/src/test/scala/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/log4j", From 1c3256bbfd022ed75c73d66cc7ea8a04ebee1ccf Mon Sep 17 00:00:00 2001 From: Lily He Date: Thu, 8 Jul 2021 20:46:31 +0000 Subject: [PATCH 276/409] util|finagle|finatra|twitter-server|scrooge: fix typo in CHANGELOG Problem/Solution: There is a typo in the Changelogs. The ScalaCheck version was updated, not Scalatest. Fixed the typo. Differential Revision: https://phabricator.twitter.biz/D701638 --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b90f47a7..0f261080 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,7 +9,7 @@ Unreleased Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ -* Update Scalatest to version 1.15.4 ``PHAB_ID=D691691`` +* Update ScalaCheck to version 1.15.4 ``PHAB_ID=D691691`` 21.6.0 ------ From fd2b05f08989f5d651cee30073c64225bcaddc2e Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Fri, 9 Jul 2021 00:47:19 +0000 Subject: [PATCH 277/409] {util-stats,finagle-stats-core}: Use new style of metadata to construct expressions Problem We now have the new style of using metadata, but we're not using it anywhere Solution Strip the `kernel` hack from MetricBuilder, and use the new style of grabbing metadata from concrete metric instances. Differential Revision: https://phabricator.twitter.biz/D674956 --- .../com/twitter/server/handler/MetricExpressionHandler.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala b/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala index a12d8537..b573da3d 100644 --- a/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala @@ -9,7 +9,8 @@ import com.twitter.finagle.stats.exp.{ Expression, FunctionExpression, HistogramExpression, - MetricExpression + MetricExpression, + NoExpression } import com.twitter.finagle.stats.{MetricBuilder, StatsFormatter, metadataScopeSeparator} import com.twitter.io.Buf @@ -35,6 +36,7 @@ object MetricExpressionHandler { case ConstantExpression(repr) => repr case FunctionExpression(funcName, exprs) => s"$funcName(${exprs.map { expr => translateToQuery(expr, latched) }.mkString(",")})" + case NoExpression => "null" } // Form a fully formatted name of the histogram with components From 93121600ffe8e8a68b0eed1e5763ab93366082d3 Mon Sep 17 00:00:00 2001 From: Joy Bestourous Date: Fri, 9 Jul 2021 19:29:53 +0000 Subject: [PATCH 278/409] util: make CounterishGauge its own MetricType Problem/Solution: We currently track if a metric is a counter-ish gauge via a boolean isCounterishGauge. Let's instead treat it as its own metric type. JIRA Issues: CSL-11006 Differential Revision: https://phabricator.twitter.biz/D700353 --- CHANGELOG.rst | 8 ++++++++ .../handler/MetricMetadataQueryHandler.scala | 2 +- .../server/util/MetricSchemaSerializer.scala | 12 +++++++----- .../handler/MetricMetadataQueryHandlerTest.scala | 4 ++-- .../server/util/MetricSchemaJsonModuleTest.scala | 14 ++++++++++---- 5 files changed, 28 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0f261080..ddb18b0c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,8 +7,16 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Admin Endpoint Versions +~~~~~~~~~~~~~~~~~~~~~~~ + +* Bump metric_metadata.json to version 3.2: CounterishGauge now exports with + "kind": "counterish_gauge", instead of "counterish_gauge": "true" + ``PHAB_ID=D700353``. + Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ + * Update ScalaCheck to version 1.15.4 ``PHAB_ID=D691691`` 21.6.0 diff --git a/server/src/main/scala/com/twitter/server/handler/MetricMetadataQueryHandler.scala b/server/src/main/scala/com/twitter/server/handler/MetricMetadataQueryHandler.scala index 84204958..23af1267 100644 --- a/server/src/main/scala/com/twitter/server/handler/MetricMetadataQueryHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/MetricMetadataQueryHandler.scala @@ -119,7 +119,7 @@ class MetricMetadataQueryHandler(source: MetricSchemaSource = new MetricSchemaSo content = Buf.Utf8( AdminJsonConverter.writeToString( Map( - "@version" -> 3.1, + "@version" -> 3.2, "counters_latched" -> latched, "separator_char" -> metadataScopeSeparator(), "metrics" -> metrics diff --git a/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala b/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala index 65eb6ad9..0b2180e2 100644 --- a/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala +++ b/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala @@ -3,7 +3,12 @@ package com.twitter.server.util import com.fasterxml.jackson.core.JsonGenerator import com.fasterxml.jackson.databind.SerializerProvider import com.fasterxml.jackson.databind.ser.std.StdSerializer -import com.twitter.finagle.stats.MetricBuilder.{CounterType, GaugeType, HistogramType} +import com.twitter.finagle.stats.MetricBuilder.{ + CounterType, + CounterishGaugeType, + GaugeType, + HistogramType +} import com.twitter.finagle.stats.{MetricBuilder, StatsFormatter, metadataScopeSeparator} object SchemaSerializer extends StdSerializer[MetricBuilder](classOf[MetricBuilder]) { @@ -40,6 +45,7 @@ object SchemaSerializer extends StdSerializer[MetricBuilder](classOf[MetricBuild jsonGenerator.writeEndArray() val dataType = metricBuilder.metricType match { case CounterType => "counter" + case CounterishGaugeType => "counterish_gauge" case GaugeType => "gauge" case HistogramType => "histogram" } @@ -57,10 +63,6 @@ object SchemaSerializer extends StdSerializer[MetricBuilder](classOf[MetricBuild jsonGenerator.writeBooleanField("key_indicator", metricBuilder.keyIndicator) metricBuilder.metricType match { - case GaugeType => - if (metricBuilder.isCounterishGauge) { - jsonGenerator.writeBooleanField("counterish_gauge", true) - } case HistogramType => jsonGenerator.writeObjectFieldStart("buckets") jsonGenerator.writeStringField("count", statsFormatter.histogramSeparator + "count") diff --git a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala index 090490ad..19cbca07 100644 --- a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala @@ -347,7 +347,7 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { val responseStart = """ | { - | "@version" : 3.1, + | "@version" : 3.2, | "counters_latched" : true, | "separator_char" : "/", """.stripMargin @@ -361,7 +361,7 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { val responseStart = """ | { - | "@version" : 3.1, + | "@version" : 3.2, | "counters_latched" : false, | "separator_char" : "/", """.stripMargin diff --git a/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala b/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala index 627503e4..5e0ba0d5 100644 --- a/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala +++ b/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala @@ -2,7 +2,12 @@ package com.twitter.server.util import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.scala.{DefaultScalaModule, ScalaObjectMapper} -import com.twitter.finagle.stats.MetricBuilder.{CounterType, GaugeType, HistogramType} +import com.twitter.finagle.stats.MetricBuilder.{ + CounterType, + CounterishGaugeType, + GaugeType, + HistogramType +} import com.twitter.finagle.stats._ import org.scalatest.funsuite.AnyFunSuite @@ -112,14 +117,15 @@ class MetricSchemaJsonModuleTest extends AnyFunSuite { )) } - test("GaugeSchema does not include counterishGauge unless it is true") { + test("GaugeSchema and CounterishGaugeSchema are recorded as different types") { val counterish = gaugeSchema.withCounterishGauge for (schema <- Seq(gaugeSchema, counterish)) { val serializedString = AdminJsonConverter.writeToString(schema) val jsonMap = jsonStrToMap(serializedString) - val expected = if (schema.isCounterishGauge) Some(true) else None - assert(jsonMap.get("counterish_gauge") == expected) + val expected = + if (schema.metricType == CounterishGaugeType) Some("counterish_gauge") else Some("gauge") + assert(jsonMap.get("kind") == expected) } } From b1c3bdec2b0df4a851201c8647673f319c9b0ad5 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Tue, 13 Jul 2021 17:03:20 +0000 Subject: [PATCH 279/409] twitter-server: Change the default behavior for rating of expressions Problem We use latching_style as a work-around so that callers don't need to also understand whether each metric is a counter or not, and whether the counters are latched or not. However, in the long run we would like to switch to not reporting rated-ness, so we can instead rely on the callers understanding the metrics in a deeper way. Currently, the "rated" style is the default, and it will show the "true" latch style if you pass in "latching_style=true". Solution Update "not rated" to be the default. Result We will only rate metrics if it is appropriate for them to be rated (ie they are unlatched) and latching_style=true is passed in. JIRA Issues: CSL-11145 Differential Revision: https://phabricator.twitter.biz/D702897 --- .../handler/MetricExpressionHandler.scala | 18 +++++----- .../exp/MetricExpressionHandlerTest.scala | 33 ++++++++++--------- 2 files changed, 28 insertions(+), 23 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala b/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala index b573da3d..4a7443bb 100644 --- a/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala @@ -26,16 +26,16 @@ object MetricExpressionHandler { /** * Translate the [[Expression]] object to a single line string which represents generic * query language. - * @param latched For unlatched counter, we wrap the metric within `rate` + * @param shouldRate If true, we wrap the metric with `rate` */ // exposed for testing - private[server] def translateToQuery(expr: Expression, latched: Boolean = false): String = + private[server] def translateToQuery(expr: Expression, shouldRate: Boolean): String = expr match { case HistogramExpression(schema, component) => getHisto(schema, component) - case MetricExpression(schema) => getMetric(schema, latched) + case MetricExpression(schema) => getMetric(schema, shouldRate) case ConstantExpression(repr) => repr case FunctionExpression(funcName, exprs) => - s"$funcName(${exprs.map { expr => translateToQuery(expr, latched) }.mkString(",")})" + s"$funcName(${exprs.map { expr => translateToQuery(expr, shouldRate) }.mkString(",")})" case NoExpression => "null" } @@ -61,10 +61,10 @@ object MetricExpressionHandler { // Form metrics other than histograms, rate() for unlatched counters private def getMetric( metricBuilder: MetricBuilder, - latched: Boolean, + shouldRate: Boolean, ): String = { metricBuilder.metricType match { - case CounterType if !latched => + case CounterType if shouldRate => s"rate(${metricBuilder.name.mkString(metadataScopeSeparator())})" case other => metricBuilder.name.mkString(metadataScopeSeparator()) } @@ -105,9 +105,11 @@ class MetricExpressionHandler(source: MetricSchemaSource = new MetricSchemaSourc // name match if name param is present && (nameParam.isEmpty || nameParam.contains(expressionSchema.name))) - val latched = latchParam.exists { value => value == "true" || value == "1" } && sourceLatched + val shouldRate = latchParam.exists { value => + value == "true" || value == "1" + } && !sourceLatched val expressions = filteredSchemas.map { expressionSchema => - expressionSchema.copy(exprQuery = translateToQuery(expressionSchema.expr, latched)) + expressionSchema.copy(exprQuery = translateToQuery(expressionSchema.expr, shouldRate)) } newResponse( diff --git a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala index 0aa1818b..e0c8e50d 100644 --- a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala @@ -93,7 +93,7 @@ class MetricExpressionHandlerTest extends AnyFunSuite { .filter(m => m.getOrElse("name", "") == "success_rate").head.getOrElse("expression", "") } - test("Get the all expressions") { + test("Get all expressions") { val expectedResponse = """ |{ @@ -108,7 +108,7 @@ class MetricExpressionHandlerTest extends AnyFunSuite { | "service_name" : "Unspecified", | "role" : "NoRoleSpecified" | }, - | "expression" : "multiply(100.0,divide(rate(success),plus(rate(success),rate(failures))))", + | "expression" : "multiply(100.0,divide(success,plus(success,failures)))", | "bounds" : { | "kind" : "monotone", | "operator" : ">", @@ -128,7 +128,7 @@ class MetricExpressionHandlerTest extends AnyFunSuite { | "role" : "NoRoleSpecified" | }, | "namespaces" : ["path","to","tenantName"], - | "expression" : "plus(rate(success),rate(failures))", + | "expression" : "plus(success,failures)", | "bounds" : { | "kind" : "unbounded" | }, @@ -169,31 +169,34 @@ class MetricExpressionHandlerTest extends AnyFunSuite { test("Get the latched expression without latched_style") { val responseString = await(latchedHandler(testRequest)).contentString - assert(getSucessRateExpression( - responseString) == "multiply(100.0,divide(rate(success),plus(rate(success),rate(failures))))") + assert( + getSucessRateExpression( + responseString) == "multiply(100.0,divide(success,plus(success,failures)))") } test("translate expressions - counters") { val latchedResult = - MetricExpressionHandler.translateToQuery(successRateExpression.expr, latched = true) + MetricExpressionHandler.translateToQuery(successRateExpression.expr, shouldRate = false) assert(latchedResult == "multiply(100.0,divide(success,plus(success,failures)))") val unlatchedResult = - MetricExpressionHandler.translateToQuery(successRateExpression.expr) + MetricExpressionHandler.translateToQuery(successRateExpression.expr, shouldRate = true) assert( unlatchedResult == "multiply(100.0,divide(rate(success),plus(rate(success),rate(failures))))") } test("translate histogram expressions - latched does not affect result") { - val latchedResult = MetricExpressionHandler.translateToQuery(latencyP99.expr, latched = true) - val unLatchedResult = MetricExpressionHandler.translateToQuery(latencyP99.expr) + val latchedResult = + MetricExpressionHandler.translateToQuery(latencyP99.expr, shouldRate = false) + val unLatchedResult = + MetricExpressionHandler.translateToQuery(latencyP99.expr, shouldRate = true) assert(latchedResult == unLatchedResult) } test("translate histogram expressions - components") { val latencyMinExpr = Expression(latencyMb, Left(Expression.Min)) - val min = MetricExpressionHandler.translateToQuery(latencyMinExpr) - val p99 = MetricExpressionHandler.translateToQuery(latencyP99.expr) + val min = MetricExpressionHandler.translateToQuery(latencyMinExpr, shouldRate = false) + val p99 = MetricExpressionHandler.translateToQuery(latencyP99.expr, shouldRate = false) assert(min == "latency.min") assert(p99 == "latency.p99") } @@ -201,7 +204,7 @@ class MetricExpressionHandlerTest extends AnyFunSuite { test("translate expressions - gauges") { val connMb = MetricBuilder(name = Seq("client", "connections"), metricType = GaugeType, statsReceiver = sr) - val result = MetricExpressionHandler.translateToQuery(Expression(connMb)) + val result = MetricExpressionHandler.translateToQuery(Expression(connMb), shouldRate = false) assert(result == "client/connections") } @@ -220,7 +223,7 @@ class MetricExpressionHandlerTest extends AnyFunSuite { | "service_name" : "Unspecified", | "role" : "NoRoleSpecified" | }, - | "expression" : "multiply(100.0,divide(rate(success),plus(rate(success),rate(failures))))", + | "expression" : "multiply(100.0,divide(success,plus(success,failures)))", | "bounds" : { | "kind" : "monotone", | "operator" : ">", @@ -256,7 +259,7 @@ class MetricExpressionHandlerTest extends AnyFunSuite { | "role" : "NoRoleSpecified" | }, | "namespaces" : ["path","to","tenantName"], - | "expression" : "plus(rate(success),rate(failures))", + | "expression" : "plus(success,failures)", | "bounds" : { | "kind" : "unbounded" | }, @@ -333,7 +336,7 @@ class MetricExpressionHandlerTest extends AnyFunSuite { | "role" : "NoRoleSpecified" | }, | "namespaces" : ["path","to","tenantName"], - | "expression" : "plus(rate(success),rate(failures))", + | "expression" : "plus(success,failures)", | "bounds" : { | "kind" : "unbounded" | }, From b65fa2e03cad44f41f024982c225ec0a3c4b7cbb Mon Sep 17 00:00:00 2001 From: Dorothy Ordogh Date: Mon, 19 Jul 2021 19:53:47 +0000 Subject: [PATCH 280/409] finagle|finatra|scrooge|twitter-server|util: 3rd party upgrade cadence note Problem/Solution We agreed to upgrade Netty, Guice, Jackson, Caffeine, & sbt on a quarterly cadence. We should communicate this in our `CONTRIBUTING.md` docs. JIRA Issues: CSL-11166 Differential Revision: https://phabricator.twitter.biz/D704594 --- CONTRIBUTING.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 76bd6bf3..c75c6946 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -35,6 +35,12 @@ If you have any questions or run into any problems, please create an issue here, chat with us in [gitter](https://gitter.im/twitter/finagle), or email the Finaglers [mailing list](https://groups.google.com/forum/#!forum/finaglers). +### 3rd party upgrades + +We upgrade the following 3rd party libraries/tools at least once every 3 months: +* [sbt](https://github.com/sbt/sbt) +* [Jackson](https://github.com/FasterXML/jackson) + ## Workflow The workflow that we support: From 5d1e5767432e2286d353887af1e45b5b8d0dad81 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Tue, 20 Jul 2021 01:13:00 +0000 Subject: [PATCH 281/409] finagle-core: Standard Service Metrics for thrift services Problem Currently, top-line service metrics include customized name snippets, potentially different counter types, and different separators, causing fleet queries with many challenges. Goals 1. Use internally consistent patterns. 2. Use forward slashes to separate scopes. 3. Not use latching / counter deltas. 4. Use ThriftExceptionsAsFailures to incorporate app-level failures. Solution 1. The StandardStatsReceiver is private and can only be scoped/named in the library, there are no customized labels taken in place. 2. Use a hard-coded separator for standard metrics. 3. Introduce a new MetricType - UnlatchedCounter for standard counters, need to collaborate with MetricsBackend to correctly aggregate these counters. 4. Inject the StandardStatsReceiver and a ResponseClassifier into the server stack parameters. Result Using a different ResponseClassifier (for thrift services that are not configured with it) determines that we need to have a set of new metrics instead of a StatsReceiver with a special Broadcast Counter/Stat/Gauge. Added overall RPC metrics that aggregate all servers' metrics in an application. Since Http service uses its own StatsFilter, we will address it in a separate revision. JIRA Issues: CSL-11128 Differential Revision: https://phabricator.twitter.biz/D697720 --- .../com/twitter/server/util/MetricSchemaSerializer.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala b/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala index 0b2180e2..a59161f2 100644 --- a/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala +++ b/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala @@ -7,7 +7,8 @@ import com.twitter.finagle.stats.MetricBuilder.{ CounterType, CounterishGaugeType, GaugeType, - HistogramType + HistogramType, + UnlatchedCounter } import com.twitter.finagle.stats.{MetricBuilder, StatsFormatter, metadataScopeSeparator} @@ -44,6 +45,7 @@ object SchemaSerializer extends StdSerializer[MetricBuilder](classOf[MetricBuild } jsonGenerator.writeEndArray() val dataType = metricBuilder.metricType match { + case UnlatchedCounter => "unlatched_counter" case CounterType => "counter" case CounterishGaugeType => "counterish_gauge" case GaugeType => "gauge" From 1dd568e68ee273b2e53b08a67ad188796aa1a21f Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Wed, 28 Jul 2021 18:39:25 +0000 Subject: [PATCH 282/409] oss/github: Use codecov github action Problem/Solution Instead of curling and executing a script, integrating the codecov recommended github action. https://github.com/codecov/codecov-action JIRA Issues: CSL-11181 Differential Revision: https://phabricator.twitter.biz/D710862 --- .github/workflows/continuous-integration.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index a0474700..a33e51f6 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -62,8 +62,6 @@ jobs: - name: test run: ${{ format('./sbt ++{0} clean coverage test coverageReport', matrix.scala) }} - name: coverage - if: ${{ success() }} run: ${{ format('./sbt ++{0} coverageAggregate', matrix.scala) }} - name: upload coverage - if: ${{ success() }} - run: unset TRAVIS; bash <(curl -s https://codecov.io/bash) + uses: codecov/codecov-action@v2 From 368652032bcc6cc6355f00966439d541680afe21 Mon Sep 17 00:00:00 2001 From: Hamdi Allam Date: Fri, 30 Jul 2021 20:52:22 +0000 Subject: [PATCH 283/409] Twitter-oss: Prepare OSS libraries for release 21.7.0 Problem We want to release the next versions of our Twitter OSS libraries 21.7.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-11162 Differential Revision: https://phabricator.twitter.biz/D713326 --- CHANGELOG.rst | 3 +++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ddb18b0c..230f610a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +21.7.0 +------ + Admin Endpoint Versions ~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/README.md b/README.md index 90042271..d4a12216 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index c46b0bc4..e97b83ad 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.7.0-SNAPSHOT" +val releaseVersion = "21.7.0" val jacksonVersion = "2.11.2" val jacksonLibs = Seq( From 971d346c31a8c44a317dd8c96298785ed46e381f Mon Sep 17 00:00:00 2001 From: Hamdi Allam Date: Fri, 30 Jul 2021 21:57:03 +0000 Subject: [PATCH 284/409] Twitter-oss: update OSS libraries post-release to 21.8.0 Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 21.8.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. JIRA Issues: CSL-11162 Differential Revision: https://phabricator.twitter.biz/D715563 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d4a12216..90042271 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index e97b83ad..01645fd2 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.7.0" +val releaseVersion = "21.8.0-SNAPSHOT" val jacksonVersion = "2.11.2" val jacksonLibs = Seq( From 4cda5ed07a36075690d238cf97b32122ec16d5dc Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Fri, 30 Jul 2021 22:04:26 +0000 Subject: [PATCH 285/409] util-stats,finagle,twitter-server: Labels want to be free Problem Right now, it's difficult to add new labels to metrics, because they're hardcoded in a case class. Solution Make labels just a simple Map[String, String], or a dictionary when we export it. JIRA Issues: CSL-11163 Differential Revision: https://phabricator.twitter.biz/D704479 --- CHANGELOG.rst | 5 +++++ doc/src/sphinx/Admin.rst | 11 ++++++----- .../handler/MetricExpressionHandler.scala | 2 +- .../server/util/exp/ExpressionJson.scala | 19 ++++++++++++------- .../exp/MetricExpressionHandlerTest.scala | 16 ++++++++-------- 5 files changed, 32 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 230f610a..13e5e12a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -17,6 +17,11 @@ Admin Endpoint Versions "kind": "counterish_gauge", instead of "counterish_gauge": "true" ``PHAB_ID=D700353``. +* Bump expressions.json to version 1.1: the `labels` field in the Metric + Metadata Expressions output to return a dictionary instead of a well-defined + JSON object. However, the existing fields in `labels` will be preserved for + now. ``PHAB_ID=D700065`` + Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/src/sphinx/Admin.rst b/doc/src/sphinx/Admin.rst index e7ab4245..04863f3d 100644 --- a/doc/src/sphinx/Admin.rst +++ b/doc/src/sphinx/Admin.rst @@ -517,17 +517,18 @@ Top-level/instance-wide Fields: Expression Object Fields: ^^^^^^^^^^^^^^^^^^^^^^^^^ :name: a string representing the name or significance of the expression. -:labels: an object which describes where the metric comes from (see next section for labels-level fields). +:labels: a dictionary (string -> string) with additional metadata about the metric :expresssion: a string representation illustrating how to aggregate metrics into the expression value. :bounds: an object which describes the bounds for what values resulting from the expression should be considered "healthy" for the service. :description: text description of the expression, intended for human consumption. :unit: the appropriate unit for this metric (ex, milliseconds, megabytes, count). -Labels Object Fields: -^^^^^^^^^^^^^^^^^^^^^ -This object is likely to change content and even become more flexible in the future. It is safest to treat it as a Map[String, String]. +Labels Entries: +^^^^^^^^^^^^^^^ +Although it should be interpreted as a dictionary, not as an object, there are some common fields that are often relevant, especially for Finagle services. + :process_path: a string indicating the relevant downstream, when there is one. -:service_name: a string representing the finagle service name. particularly helpful for services containing multiple internal servers (ex, thrift and http). +:service_name: a string representing the Finagle service name. particularly helpful for services containing multiple internal servers (ex, thrift and http). :role: either "Server" or "Client" indicating whether the expression pertains to this server or one of its downstreams. Should be treated as a string. Bounds Object Fields: diff --git a/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala b/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala index 4a7443bb..295e91cf 100644 --- a/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala @@ -20,7 +20,7 @@ import com.twitter.server.util.{AdminJsonConverter, MetricSchemaSource} import com.twitter.util.Future object MetricExpressionHandler { - private val Version = 1.0 + private val Version = 1.1 private val statsFormatter = StatsFormatter.default /** diff --git a/server/src/main/scala/com/twitter/server/util/exp/ExpressionJson.scala b/server/src/main/scala/com/twitter/server/util/exp/ExpressionJson.scala index deef21dd..bf5d3bda 100644 --- a/server/src/main/scala/com/twitter/server/util/exp/ExpressionJson.scala +++ b/server/src/main/scala/com/twitter/server/util/exp/ExpressionJson.scala @@ -4,6 +4,7 @@ import com.fasterxml.jackson.core.{JsonGenerator, JsonParser} import com.fasterxml.jackson.databind.deser.std.StdDeserializer import com.fasterxml.jackson.databind.ser.std.StdSerializer import com.fasterxml.jackson.databind.{DeserializationContext, SerializerProvider} +import com.twitter.finagle.stats.NoRoleSpecified import com.twitter.finagle.stats.exp._ /** @@ -26,13 +27,17 @@ object ExpressionJson { gen.writeStringField("name", expressionSchema.name) gen.writeObjectFieldStart("labels") - gen.writeStringField( - "process_path", - expressionSchema.labels.processPath.getOrElse("Unspecified")) - gen.writeStringField( - "service_name", - expressionSchema.labels.serviceName.getOrElse("Unspecified")) - gen.writeStringField("role", expressionSchema.labels.role.toString) + + // we need this to evolve the expressions format compatibly + // in the future, we will only export labels that are set + val labels = Map( + ExpressionSchema.ProcessPath -> "Unspecified", + ExpressionSchema.ServiceName -> "Unspecified", + ExpressionSchema.Role -> NoRoleSpecified.toString + ) ++ expressionSchema.labels + for ((key, value) <- labels) { + gen.writeStringField(key, value) + } gen.writeEndObject() if (expressionSchema.namespace.nonEmpty) { diff --git a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala index e0c8e50d..729e0855 100644 --- a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala @@ -44,9 +44,9 @@ class MetricExpressionHandlerTest extends AnyFunSuite { ExpressionSchema("latency_p99", Expression(latencyMb, Right(0.99))).withNamespace("tenantName") val expressionSchemaMap: Map[ExpressionSchemaKey, ExpressionSchema] = Map( - ExpressionSchemaKey("success_rate", None, Seq()) -> successRateExpression, - ExpressionSchemaKey("throughput", None, Seq()) -> throughputExpression, - ExpressionSchemaKey("latency", None, Seq("path", "to", "tenantName")) -> latencyP99 + ExpressionSchemaKey("success_rate", Map(), Seq()) -> successRateExpression, + ExpressionSchemaKey("throughput", Map(), Seq()) -> throughputExpression, + ExpressionSchemaKey("latency", Map(), Seq("path", "to", "tenantName")) -> latencyP99 ) val expressionRegistry = new SchemaRegistry { @@ -97,7 +97,7 @@ class MetricExpressionHandlerTest extends AnyFunSuite { val expectedResponse = """ |{ - | "@version" : 1.0, + | "@version" : 1.1, | "counters_latched" : false, | "separator_char" : "/", | "expressions" : [ @@ -212,7 +212,7 @@ class MetricExpressionHandlerTest extends AnyFunSuite { val expectedResponse = """ |{ - | "@version" : 1.0, + | "@version" : 1.1, | "counters_latched" : false, | "separator_char" : "/", | "expressions" : [ @@ -247,7 +247,7 @@ class MetricExpressionHandlerTest extends AnyFunSuite { val expectedResponse = """ |{ - | "@version" : 1.0, + | "@version" : 1.1, | "counters_latched" : false, | "separator_char" : "/", | "expressions" : [ @@ -293,7 +293,7 @@ class MetricExpressionHandlerTest extends AnyFunSuite { val expectedResponse = """ |{ - | "@version" : 1.0, + | "@version" : 1.1, | "counters_latched" : false, | "separator_char" : "/", | "expressions" : [ @@ -324,7 +324,7 @@ class MetricExpressionHandlerTest extends AnyFunSuite { val expectedResponse = """ |{ - | "@version" : 1.0, + | "@version" : 1.1, | "counters_latched" : false, | "separator_char" : "/", | "expressions" : [ From d6f66d78be00dc946b3e0a2fd79d5d3275a77d7f Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Fri, 30 Jul 2021 22:36:15 +0000 Subject: [PATCH 286/409] twitter-server: Updated the "too many statsreceivers" lint error to recommend loadServiceDenied Problem The lint error recommends using loadServiceIgnoredPaths which is not very fashionable. Solution Instead recommend loadServiceDenied, which is what the cool kids use!! JIRA Issues: CSL-11218 Differential Revision: https://phabricator.twitter.biz/D714715 --- .../com/twitter/server/lint/NumberOfStatsReceiversRule.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/scala/com/twitter/server/lint/NumberOfStatsReceiversRule.scala b/server/src/main/scala/com/twitter/server/lint/NumberOfStatsReceiversRule.scala index df991c2f..59ff52ce 100644 --- a/server/src/main/scala/com/twitter/server/lint/NumberOfStatsReceiversRule.scala +++ b/server/src/main/scala/com/twitter/server/lint/NumberOfStatsReceiversRule.scala @@ -16,7 +16,7 @@ object NumberOfStatsReceiversRule { "More than one StatsReceiver loaded causes a larger than necessary " + "memory footprint and slower runtime usage. Examine your (transitive) " + "dependencies and remove unwanted StatsReceivers either via dependency " + - "management or the com.twitter.finagle.util.loadServiceIgnoredPaths flag. " + + "management or the com.twitter.finagle.util.loadServiceDenied flag. " + "Alternatively, having none loaded indicates that the service will not " + "have any telemetry reported which is dangerous way to operate a service." ) { From 3ad5628372f5399562a6da8dcc7c097b35e41272 Mon Sep 17 00:00:00 2001 From: Hamdi Allam Date: Wed, 11 Aug 2021 20:31:06 +0000 Subject: [PATCH 287/409] Twitter-oss: Prepare OSS libraries for release 21.8.0 Problem We want to release the next versions of our Twitter OSS libraries 21.8.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-11162 Differential Revision: https://phabricator.twitter.biz/D722632 --- CHANGELOG.rst | 7 ++++++- README.md | 2 +- build.sbt | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 13e5e12a..e590cb07 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,9 +7,14 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- -21.7.0 +21.8.0 ------ +No Changes + +21.7.0 (No Release) +------------------- + Admin Endpoint Versions ~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/README.md b/README.md index 90042271..d4a12216 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 01645fd2..c5298d49 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.8.0-SNAPSHOT" +val releaseVersion = "21.8.0" val jacksonVersion = "2.11.2" val jacksonLibs = Seq( From 90cc3ccbe3406e153c7f07bf74d03c460fdaa632 Mon Sep 17 00:00:00 2001 From: Hamdi Allam Date: Fri, 13 Aug 2021 00:36:16 +0000 Subject: [PATCH 288/409] Twitter-oss: Prepare OSS libraries for release 21.8.0 Problem We want to release the next versions of our Twitter OSS libraries 21.8.0 util scrooge finagle twitter-server finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-11162 Differential Revision: https://phabricator.twitter.biz/D723532 --- CHANGELOG.rst | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e590cb07..418fd106 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,13 +7,8 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- -21.8.0 ------- - -No Changes - -21.7.0 (No Release) -------------------- +21.8.0 (No 21.7.0 Release) +-------------------------- Admin Endpoint Versions ~~~~~~~~~~~~~~~~~~~~~~~ From 53c857447b77862bf9f72f7525f6a922b90e282d Mon Sep 17 00:00:00 2001 From: Hamdi Allam Date: Fri, 13 Aug 2021 00:54:10 +0000 Subject: [PATCH 289/409] Twitter-oss: update OSS libraries post-release to 21.9.0 Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 21.9.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. JIRA Issues: CSL-11162 Differential Revision: https://phabricator.twitter.biz/D722646 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d4a12216..90042271 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index c5298d49..60f3b796 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.8.0" +val releaseVersion = "21.9.0-SNAPSHOT" val jacksonVersion = "2.11.2" val jacksonLibs = Seq( From 6f0cd78df3a6d7b84d5ccb54af069c855ac20c81 Mon Sep 17 00:00:00 2001 From: Hamdi Allam Date: Fri, 13 Aug 2021 16:38:26 +0000 Subject: [PATCH 290/409] Revert "Twitter-oss: update OSS libraries post-release to 21.9.0" Problem: Sonatype requires a javadoc jar, even if empty to release. Solution / Result: Revert the post-release commit to act as the new SHA to which will be released. This SHA will come after D724170 which doesn't skip the creation of the empty util javadoc JAR for Scala 3. JIRA Issues: CSL-11162 Differential Revision: https://phabricator.twitter.biz/D724302 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 90042271..d4a12216 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 60f3b796..c5298d49 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.9.0-SNAPSHOT" +val releaseVersion = "21.8.0" val jacksonVersion = "2.11.2" val jacksonLibs = Seq( From 9b7791449d8879a4d708e5634388f059beff3b71 Mon Sep 17 00:00:00 2001 From: Hamdi Allam Date: Fri, 13 Aug 2021 17:53:30 +0000 Subject: [PATCH 291/409] Twitter-oss: update OSS libraries post-release to 21.9.0 Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 21.9.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. JIRA Issues: CSL-11162 Differential Revision: https://phabricator.twitter.biz/D724396 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d4a12216..90042271 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index c5298d49..60f3b796 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.8.0" +val releaseVersion = "21.9.0-SNAPSHOT" val jacksonVersion = "2.11.2" val jacksonLibs = Seq( From b52d9b343abf73a8f84f157cda8d6fd21f0efb77 Mon Sep 17 00:00:00 2001 From: Ian Bennett Date: Fri, 13 Aug 2021 21:02:08 +0000 Subject: [PATCH 292/409] twitter-server: add test coverage to ShutdownHandlerTest Problem/Solution For the `ShutdownHandlerTest` we do not have coverage for the case where the `defaultCloseGracePeriod` is overridden. Let's modify tests to allow for testing and verifying this behavior. JIRA Issues: CSL-11231 Differential Revision: https://phabricator.twitter.biz/D717003 --- .../server/handler/ShutdownHandlerTest.scala | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/server/src/test/scala/com/twitter/server/handler/ShutdownHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/ShutdownHandlerTest.scala index abe9fcbf..2d353721 100644 --- a/server/src/test/scala/com/twitter/server/handler/ShutdownHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/ShutdownHandlerTest.scala @@ -1,9 +1,9 @@ package com.twitter.server.handler import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.http.{Status, Request, Method} +import com.twitter.finagle.http.{Method, Request, Status} import com.twitter.server.TwitterServer -import com.twitter.util.{Await, Awaitable, Closable, Future, Time} +import com.twitter.util.{Await, Awaitable, Closable, Duration, Future, Time} import org.scalatest.funsuite.AnyFunSuite class ShutdownHandlerTest extends AnyFunSuite { @@ -11,6 +11,10 @@ class ShutdownHandlerTest extends AnyFunSuite { private[this] def await[T](a: Awaitable[T]): T = Await.result(a, 2.seconds) class Closer(testDeadline: Time => Unit) extends TwitterServer { + + @volatile var defaultGraceOverride = Duration.Zero + override def defaultCloseGracePeriod: Duration = defaultGraceOverride + @volatile var closed = false closeOnExit { Closable.make { deadline => @@ -37,6 +41,21 @@ class ShutdownHandlerTest extends AnyFunSuite { assert(closer.closed) }) + test("close without a grace period, but closer overrides default grace")( + Time.withCurrentTimeFrozen { tc => + val grace = 5.seconds + val now = Time.now + val closer = Closer.mk { deadline => + // MinGrace is 1 second + assert(deadline == now + grace) + } + closer.defaultGraceOverride = grace + val handler = new ShutdownHandler(closer) + val rsp = await(handler(Request(Method.Post, "/foo"))) + assert(rsp.status == Status.Ok) + assert(closer.closed) + }) + test("close with a grace period") { val grace = 10.seconds val expectedDeadline = Time.now + grace From 62b32593995d49c9fe5e049f5d2a5b60710bff1a Mon Sep 17 00:00:00 2001 From: Hannah Ku Date: Tue, 17 Aug 2021 21:28:52 +0000 Subject: [PATCH 293/409] [tfe limiter server] mark bazel-compatible Differential Revision: https://phabricator.twitter.biz/D726504 --- slf4j-jdk14/src/main/scala/BUILD | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/slf4j-jdk14/src/main/scala/BUILD b/slf4j-jdk14/src/main/scala/BUILD index a10cca07..3f10e49a 100644 --- a/slf4j-jdk14/src/main/scala/BUILD +++ b/slf4j-jdk14/src/main/scala/BUILD @@ -8,7 +8,10 @@ scala_library( ), scope = "forced", # Exposes a service-loaded class. strict_deps = True, - tags = ["logging_impl_check_whitelisted_target"], + tags = [ + "bazel-compatible", + "logging_impl_check_whitelisted_target", + ], dependencies = [ "3rdparty/jvm/org/slf4j:slf4j-api", scoped( From b4a8b44f34445d31ebfcfa00f0343f79cbf27089 Mon Sep 17 00:00:00 2001 From: Matthew Dickinson Date: Fri, 20 Aug 2021 15:15:19 +0000 Subject: [PATCH 294/409] 3rdparty: bump jackson version to 2.11.4 Problem We're on an old version of jackson Solution Upgrade!! Differential Revision: https://phabricator.twitter.biz/D727879 --- CHANGELOG.rst | 5 +++++ build.sbt | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 418fd106..c6efbee0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Runtime Behavior Changes +~~~~~~~~~~~~~~~~~~~~~~~~ + +* Bump version of Jackson to 2.11.4. ``PHAB_ID=D727879`` + 21.8.0 (No 21.7.0 Release) -------------------------- diff --git a/build.sbt b/build.sbt index 60f3b796..b598a261 100644 --- a/build.sbt +++ b/build.sbt @@ -6,7 +6,7 @@ Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch val releaseVersion = "21.9.0-SNAPSHOT" -val jacksonVersion = "2.11.2" +val jacksonVersion = "2.11.4" val jacksonLibs = Seq( "com.fasterxml.jackson.core" % "jackson-core" % jacksonVersion, "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion, From 0a2bfd6dee0d1651545b9ad0b001cf78a5f5b28d Mon Sep 17 00:00:00 2001 From: Matthew Dickinson Date: Tue, 24 Aug 2021 18:31:17 +0000 Subject: [PATCH 295/409] logback: bump to 1.2.5 Problem We're on an old version of logback Solution Upgrade!! Differential Revision: https://phabricator.twitter.biz/D729767 --- CHANGELOG.rst | 2 ++ build.sbt | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c6efbee0..a9b6df05 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -12,6 +12,8 @@ Runtime Behavior Changes * Bump version of Jackson to 2.11.4. ``PHAB_ID=D727879`` +* Bump version of logback to 1.2.5. ``PHAB_ID=D729767`` + 21.8.0 (No 21.7.0 Release) -------------------------- diff --git a/build.sbt b/build.sbt index b598a261..f573355e 100644 --- a/build.sbt +++ b/build.sbt @@ -12,6 +12,7 @@ val jacksonLibs = Seq( "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion, "com.fasterxml.jackson.module" %% "jackson-module-scala" % jacksonVersion exclude ("com.google.guava", "guava") ) +val logbackVersion = "1.2.5" val opencensusVersion = "0.19.1" val slf4jVersion = "1.7.30" @@ -245,8 +246,8 @@ lazy val twitterServerSlf4jLogbackClassic = (project in file("logback-classic")) moduleName := "twitter-server-logback-classic", sharedSettings) .settings(libraryDependencies ++= Seq( - "ch.qos.logback" % "logback-classic" % "1.2.3" % "provided", - "ch.qos.logback" % "logback-core" % "1.2.3" % "provided", + "ch.qos.logback" % "logback-classic" % logbackVersion % "provided", + "ch.qos.logback" % "logback-core" % logbackVersion % "provided", "org.slf4j" % "slf4j-api" % slf4jVersion, "org.slf4j" % "jcl-over-slf4j" % slf4jVersion, "org.slf4j" % "jul-to-slf4j" % slf4jVersion, From 851f2c4db83e7d1a69b76d0006d3a7fcb64aa913 Mon Sep 17 00:00:00 2001 From: Hamdi Allam Date: Thu, 2 Sep 2021 23:30:59 +0000 Subject: [PATCH 296/409] twitter-server: case class representation for balancers.json Problem The balancers.json endpoint hand crafts the json response with a string Map. `JsonConverter` also interfaces with the jackson object mapper directly Solution Introduce an intermediary case class representation that jackson can simply serialize. Define `scalaObjectMapper`, utilizing util-jackson. BalancerJsonView is the first endpoint to use this mapper. The raw ObjectMapper will be removed after migrating all of the endpoints The mappers should also be val'd up to avoid re-creating them JIRA Issues: CSL-11287 Differential Revision: https://phabricator.twitter.biz/D734490 --- build.sbt | 10 +--- server/src/main/scala/BUILD | 8 +-- .../scala/com/twitter/server/util/Json.scala | 15 ++++-- .../server/view/BalancersJsonView.scala | 50 ++++++++++++------- .../com/twitter/server/util/JsonUtils.scala | 16 ++---- .../server/view/BalancersJsonViewTest.scala | 47 +++++++++++++++++ 6 files changed, 100 insertions(+), 46 deletions(-) create mode 100644 server/src/test/scala/com/twitter/server/view/BalancersJsonViewTest.scala diff --git a/build.sbt b/build.sbt index f573355e..4f1f263f 100644 --- a/build.sbt +++ b/build.sbt @@ -6,12 +6,6 @@ Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch val releaseVersion = "21.9.0-SNAPSHOT" -val jacksonVersion = "2.11.4" -val jacksonLibs = Seq( - "com.fasterxml.jackson.core" % "jackson-core" % jacksonVersion, - "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion, - "com.fasterxml.jackson.module" %% "jackson-module-scala" % jacksonVersion exclude ("com.google.guava", "guava") -) val logbackVersion = "1.2.5" val opencensusVersion = "0.19.1" val slf4jVersion = "1.7.30" @@ -186,14 +180,14 @@ lazy val twitterServer = (project in file("server")) finagle("tunable"), util("app"), util("core"), + util("jackson"), util("jvm"), util("lint"), util("registry"), util("slf4j-api"), util("slf4j-jul-bridge"), util("tunable") - ), - libraryDependencies ++= jacksonLibs + ) ) lazy val twitterServerOpenCensus = (project in file("opencensus")) diff --git a/server/src/main/scala/BUILD b/server/src/main/scala/BUILD index 200d2614..ffba4f0b 100644 --- a/server/src/main/scala/BUILD +++ b/server/src/main/scala/BUILD @@ -9,10 +9,6 @@ scala_library( scope = "forced", # Exposes a service-loaded class. tags = ["bazel-compatible"], dependencies = [ - "3rdparty/jvm/com/fasterxml/jackson/core:jackson-annotations", - "3rdparty/jvm/com/fasterxml/jackson/core:jackson-core", - "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", - "3rdparty/jvm/com/fasterxml/jackson/module:jackson-module-scala", "3rdparty/jvm/io/netty:netty4", "3rdparty/jvm/io/netty:netty4-codec-http", "3rdparty/jvm/org/slf4j:slf4j-api", @@ -27,6 +23,7 @@ scala_library( "twitter-server/server/src/main/resources", "util/util-app/src/main/scala", "util/util-core:scala", + "util/util-jackson/src/main/scala/com/twitter/util/jackson", "util/util-jvm/src/main/scala", "util/util-lint/src/main/scala", "util/util-registry/src/main/scala", @@ -36,13 +33,12 @@ scala_library( "util/util-tunable/src/main/scala", ], exports = [ - "3rdparty/jvm/com/fasterxml/jackson/core:jackson-annotations", - "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", "3rdparty/jvm/org/slf4j:slf4j-api", "finagle/finagle-base-http/src/main", "finagle/finagle-http/src/main/scala", "util/util-app/src/main/scala", "util/util-core:scala", + "util/util-jackson/src/main/scala/com/twitter/util/jackson", "util/util-lint/src/main/scala", "util/util-slf4j-api/src/main/scala", ], diff --git a/server/src/main/scala/com/twitter/server/util/Json.scala b/server/src/main/scala/com/twitter/server/util/Json.scala index 6030b97b..8e2b68cd 100644 --- a/server/src/main/scala/com/twitter/server/util/Json.scala +++ b/server/src/main/scala/com/twitter/server/util/Json.scala @@ -12,6 +12,7 @@ import com.fasterxml.jackson.databind.{ import com.fasterxml.jackson.module.scala.DefaultScalaModule import com.twitter.finagle.http.{Response, Status, Version} import com.twitter.io.Buf +import com.twitter.util.jackson.ScalaObjectMapper import java.lang.reflect.{ParameterizedType, Type} sealed trait JsonConverterBase { @@ -22,7 +23,7 @@ sealed trait JsonConverterBase { factory.disable(JsonFactory.Feature.USE_THREAD_LOCAL_FOR_BUFFER_RECYCLING) } - private final val writer: ObjectWriter = { + private final lazy val writer: ObjectWriter = { val printer = new DefaultPrettyPrinter printer.indentArraysWith(DefaultIndenter.SYSTEM_LINEFEED_INSTANCE) @@ -63,7 +64,7 @@ sealed trait JsonConverterBase { * @note Use [[AdminJsonConverter]] for twitter-server endpoints */ object JsonConverter extends JsonConverterBase { - protected[this] def mapper: ObjectMapper = + protected[this] val mapper: ObjectMapper = new ObjectMapper(factory) .registerModule(DefaultScalaModule) } @@ -73,7 +74,15 @@ object JsonConverter extends JsonConverterBase { * consistent naming strategy. */ object AdminJsonConverter extends JsonConverterBase { - protected[this] def mapper: ObjectMapper = + // We define this writer alongside `mapper` until we migrate + // all of the admin endpoints + private[server] val prettyObjectMapper: ObjectWriter = + ScalaObjectMapper.builder + .withAdditionalJacksonModules(Seq(MetricSchemaJsonModule)) + .objectMapper(factory) + .prettyObjectMapper + + protected[this] val mapper: ObjectMapper = new ObjectMapper(factory) .registerModule(DefaultScalaModule) .registerModule(MetricSchemaJsonModule) diff --git a/server/src/main/scala/com/twitter/server/view/BalancersJsonView.scala b/server/src/main/scala/com/twitter/server/view/BalancersJsonView.scala index 0b27afcc..f3bf4ca9 100644 --- a/server/src/main/scala/com/twitter/server/view/BalancersJsonView.scala +++ b/server/src/main/scala/com/twitter/server/view/BalancersJsonView.scala @@ -3,28 +3,42 @@ package com.twitter.server.view import com.twitter.finagle.loadbalancer.Metadata import com.twitter.server.util.AdminJsonConverter -private[server] class BalancersJsonView(balancers: Seq[Metadata]) extends View { +private object BalancersJsonView { + case class Clients(clients: Seq[Balancer]) + case class Balancer(label: String, info: BalancerInfo) + case class BalancerInfo( + balancerClass: String, + status: String, + numberAvailable: Int, + numberBusy: Int, + numberClosed: Int, + totalPending: Int, + totalLoad: Double, + size: Int, + additionalInfo: Map[String, Any]) - private[view] def renderBalancer(balancer: Metadata): Map[String, Object] = - Map( - "label" -> balancer.label, - "info" -> Map( - "balancer_class" -> balancer.balancerClass, - "status" -> balancer.status, - "number_available" -> balancer.numAvailable, - "number_busy" -> balancer.numBusy, - "number_closed" -> balancer.numClosed, - "total_pending" -> balancer.totalPending, - "total_load" -> balancer.totalLoad, - "size" -> balancer.size, - "additional_info" -> balancer.additionalInfo + def convertMetadata(metadata: Metadata): Balancer = + Balancer( + label = metadata.label, + info = BalancerInfo( + balancerClass = metadata.balancerClass, + status = metadata.status, + numberAvailable = metadata.numAvailable, + numberBusy = metadata.numBusy, + numberClosed = metadata.numClosed, + totalPending = metadata.totalPending, + totalLoad = metadata.totalLoad, + size = metadata.size, + additionalInfo = metadata.additionalInfo ) ) +} + +private[server] class BalancersJsonView(balancers: Seq[Metadata]) extends View { + import BalancersJsonView._ def render: String = { - val clients = balancers.map(renderBalancer) - val asMap: Map[String, Object] = Map("clients" -> clients) - AdminJsonConverter.writeToString(asMap) + val clients = Clients(clients = balancers.map(convertMetadata)) + AdminJsonConverter.prettyObjectMapper.writeValueAsString(clients) } - } diff --git a/server/src/test/scala/com/twitter/server/util/JsonUtils.scala b/server/src/test/scala/com/twitter/server/util/JsonUtils.scala index 5562e38b..7575edce 100644 --- a/server/src/test/scala/com/twitter/server/util/JsonUtils.scala +++ b/server/src/test/scala/com/twitter/server/util/JsonUtils.scala @@ -1,17 +1,11 @@ package com.twitter.server.util -import org.scalatest -import org.scalatest.Assertions +import com.twitter.util.jackson.JsonDiff +import org.scalatest.{Assertion, Assertions} private[server] object JsonUtils { - - // NOTE: these tests assume a specific iteration order over the registries - // and HashMaps which IS NOT a guarantee. should these tests begin to fail - // due to that, we will need a more robust approach to validation. - def assertJsonResponse(actual: String, expected: String): scalatest.Assertion = { - Assertions.assert(stripWhitespace(actual) == stripWhitespace(expected)) + def assertJsonResponse(actual: String, expected: String): Assertion = { + val result = JsonDiff.diff(expected, actual) + Assertions.assert(!result.isDefined, result.toString) } - - private[this] def stripWhitespace(string: String): String = - string.filter { case c => !c.isWhitespace } } diff --git a/server/src/test/scala/com/twitter/server/view/BalancersJsonViewTest.scala b/server/src/test/scala/com/twitter/server/view/BalancersJsonViewTest.scala new file mode 100644 index 00000000..0a57f98b --- /dev/null +++ b/server/src/test/scala/com/twitter/server/view/BalancersJsonViewTest.scala @@ -0,0 +1,47 @@ +package com.twitter.server.view + +import com.twitter.finagle.{Http, Name, Service} +import com.twitter.finagle.http.{Request, Response} +import com.twitter.finagle.loadbalancer.{BalancerRegistry, Balancers} +import com.twitter.server.util.JsonUtils +import com.twitter.util.{Await, Future} +import org.scalatest.funsuite.AnyFunSuite + +class BalancersJsonViewTest extends AnyFunSuite { + + test("renders balancer json view") { + val label = "balancerjsonview" + val svc: Service[Request, Response] = Service.const(Future.value(Response())) + val client = + Http.client + .withLoadBalancer(Balancers.p2cPeakEwma()) + .newClient(Name.bound(svc), label) + + val registry = BalancerRegistry.get.allMetadata.filter(_.label == label) + assert(registry.size == 1) + + val json = (new BalancersJsonView(registry)).render + val expected = + s"""{ + | "clients" : [ + | { + | "label" : "$label", + | "info" : { + | "balancer_class" : "P2CPeakEwma", + | "status" : "Open", + | "number_available" : 1, + | "number_busy" : 0, + | "number_closed" : 0, + | "total_pending" : 0, + | "total_load" : 0.0, + | "size" : 1, + | "additional_info" : { } + | } + | } + | ] + |}""".stripMargin + + JsonUtils.assertJsonResponse(json, expected) + Await.result(client.close()) + } +} From 62f8a2d6d0a7e1616d6738371191d311dcb0020d Mon Sep 17 00:00:00 2001 From: Hamdi Allam Date: Fri, 3 Sep 2021 18:17:02 +0000 Subject: [PATCH 297/409] twitter-server: case class representation for dtab handler Problem Dtab handler hand crafts the json response Solution: Utilize util-jackson and a case class representation for the json response. Result: Add test to verify this use the scala object mapper JIRA Issues: CSL-11293 Differential Revision: https://phabricator.twitter.biz/D737280 --- .../twitter/server/handler/DtabHandler.scala | 13 ++++++-- .../server/handler/DtabHandlerTest.scala | 33 +++++++++++++++++++ 2 files changed, 43 insertions(+), 3 deletions(-) create mode 100644 server/src/test/scala/com/twitter/server/handler/DtabHandlerTest.scala diff --git a/server/src/main/scala/com/twitter/server/handler/DtabHandler.scala b/server/src/main/scala/com/twitter/server/handler/DtabHandler.scala index 1c1a3cd5..92694466 100644 --- a/server/src/main/scala/com/twitter/server/handler/DtabHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/DtabHandler.scala @@ -6,6 +6,10 @@ import com.twitter.io.Buf import com.twitter.server.util.{AdminJsonConverter, HttpUtils} import com.twitter.util.Future +private object DtabHandler { + case class Dtabs(dtab: Seq[String]) +} + /** * Dumps a simple JSON string representation of the current [[com.twitter.finagle.Dtab.base]]. * @@ -27,14 +31,17 @@ import com.twitter.util.Future * @see [[com.twitter.finagle.Dtab]] */ final class DtabHandler extends Service[Request, Response] { + import DtabHandler._ + def apply(req: Request): Future[Response] = HttpUtils.newResponse(contentType = MediaType.Json, content = Buf.Utf8(jsonResponse)) private[this] def jsonResponse: String = { - AdminJsonConverter.writeToString(Map("dtab" -> formattedDtabEntries)) + AdminJsonConverter.prettyObjectMapper.writeValueAsString(formattedDtabEntries) } - private[this] def formattedDtabEntries: Seq[String] = { - for (dentry <- Dtab.base) yield { s"${dentry.prefix.show} => ${dentry.dst.show}" } + private[this] def formattedDtabEntries: Dtabs = { + val dtabs = for (dentry <- Dtab.base) yield { s"${dentry.prefix.show} => ${dentry.dst.show}" } + Dtabs(dtabs) } } diff --git a/server/src/test/scala/com/twitter/server/handler/DtabHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/DtabHandlerTest.scala new file mode 100644 index 00000000..2865b7ba --- /dev/null +++ b/server/src/test/scala/com/twitter/server/handler/DtabHandlerTest.scala @@ -0,0 +1,33 @@ +package com.twitter.server.handler + +import com.twitter.finagle.Dtab +import com.twitter.finagle.http.Request +import com.twitter.server.util.JsonUtils +import com.twitter.util.Await +import org.scalatest.funsuite.AnyFunSuite + +class DtabHandlerTest extends AnyFunSuite { + test("renders json") { + val handler = new DtabHandler + val dtab = Dtab.read("/foo=>/bar;/baz=>/biz") + + val oldBase = Dtab.base + val json = + try { + Dtab.base = dtab + Await.result(handler(Request())).contentString + } finally { + Dtab.base = oldBase + } + + val expected = + s"""{ + | "dtab" : [ + | "/foo => /bar", + | "/baz => /biz" + | ] + |}""".stripMargin + + JsonUtils.assertJsonResponse(json, expected) + } +} From 583e7a697b6ed9adc1a31a5eb5d22940e5abd559 Mon Sep 17 00:00:00 2001 From: Helen Woldesenbet Date: Wed, 8 Sep 2021 17:38:48 +0000 Subject: [PATCH 298/409] twitter-server: ServerInfoHandler updated with ScalaObjectMapper Problem ServerInfoHandler endpoint needs to be updated to use util-jackson ScalaObjectMapper APIs Solution/Result used ScalaObjectMapper JIRA Issues: CSL-11297 Differential Revision: https://phabricator.twitter.biz/D738143 --- .../scala/com/twitter/server/handler/ServerInfoHandler.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/handler/ServerInfoHandler.scala b/server/src/main/scala/com/twitter/server/handler/ServerInfoHandler.scala index 6da4cd58..1750e897 100644 --- a/server/src/main/scala/com/twitter/server/handler/ServerInfoHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/ServerInfoHandler.scala @@ -67,8 +67,9 @@ class ServerInfoHandler() extends Service[Request, Response] { def apply(req: Request): Future[Response] = { newResponse( contentType = "application/json;charset=UTF-8", - content = - Buf.Utf8(AdminJsonConverter.writeToString(serverInfo + ("uptime" -> mxRuntime.getUptime))) + content = Buf.Utf8( + AdminJsonConverter.prettyObjectMapper.writeValueAsString( + serverInfo + ("uptime" -> mxRuntime.getUptime))) ) } } From 4a1930f0da624ff59dbe47a9508f52d98603355f Mon Sep 17 00:00:00 2001 From: Matthew Dickinson Date: Tue, 14 Sep 2021 03:54:59 +0000 Subject: [PATCH 299/409] Bump logback version to 1.2.6 Bump logback version to 1.2.6 Differential Revision: https://phabricator.twitter.biz/D742405 --- CHANGELOG.rst | 2 ++ build.sbt | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a9b6df05..a62ae870 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,8 @@ Unreleased Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ +* Bump version of Logback to 1.2.6. ``PHAB_ID=D742405`` + * Bump version of Jackson to 2.11.4. ``PHAB_ID=D727879`` * Bump version of logback to 1.2.5. ``PHAB_ID=D729767`` diff --git a/build.sbt b/build.sbt index 4f1f263f..ac29c085 100644 --- a/build.sbt +++ b/build.sbt @@ -6,7 +6,7 @@ Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch val releaseVersion = "21.9.0-SNAPSHOT" -val logbackVersion = "1.2.5" +val logbackVersion = "1.2.6" val opencensusVersion = "0.19.1" val slf4jVersion = "1.7.30" From adaa1de5ab550f6d741ebc93a02fb29c474053ce Mon Sep 17 00:00:00 2001 From: Joy Bestourous Date: Thu, 16 Sep 2021 15:08:06 +0000 Subject: [PATCH 300/409] twitter-server: rate CounterishGauges Problem: The instrumented JVM expressions aren't rated, but they should be. Solution: They're all counterish gauges, which should be rated anyway. Lets update twitter-server to include rate for CounterishGauges as well as Counters. JIRA Issues: CSL-11280 Differential Revision: https://phabricator.twitter.biz/D743620 --- .../handler/MetricExpressionHandler.scala | 22 ++++++++---- .../exp/MetricExpressionHandlerTest.scala | 35 +++++++++++++++---- 2 files changed, 43 insertions(+), 14 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala b/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala index 295e91cf..324c6ec4 100644 --- a/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala @@ -2,7 +2,7 @@ package com.twitter.server.handler import com.twitter.finagle.Service import com.twitter.finagle.http.{MediaType, Request, Response, Uri} -import com.twitter.finagle.stats.MetricBuilder.CounterType +import com.twitter.finagle.stats.MetricBuilder.{CounterType, CounterishGaugeType} import com.twitter.finagle.stats.exp.Expression._ import com.twitter.finagle.stats.exp.{ ConstantExpression, @@ -29,13 +29,17 @@ object MetricExpressionHandler { * @param shouldRate If true, we wrap the metric with `rate` */ // exposed for testing - private[server] def translateToQuery(expr: Expression, shouldRate: Boolean): String = + private[server] def translateToQuery( + expr: Expression, + shouldRate: Boolean, + sourceLatched: Boolean + ): String = expr match { case HistogramExpression(schema, component) => getHisto(schema, component) - case MetricExpression(schema) => getMetric(schema, shouldRate) + case MetricExpression(schema) => getMetric(schema, shouldRate, sourceLatched) case ConstantExpression(repr) => repr case FunctionExpression(funcName, exprs) => - s"$funcName(${exprs.map { expr => translateToQuery(expr, shouldRate) }.mkString(",")})" + s"$funcName(${exprs.map { expr => translateToQuery(expr, shouldRate, sourceLatched) }.mkString(",")})" case NoExpression => "null" } @@ -62,9 +66,12 @@ object MetricExpressionHandler { private def getMetric( metricBuilder: MetricBuilder, shouldRate: Boolean, + sourceLatched: Boolean ): String = { metricBuilder.metricType match { - case CounterType if shouldRate => + case CounterType if shouldRate && !sourceLatched => + s"rate(${metricBuilder.name.mkString(metadataScopeSeparator())})" + case CounterishGaugeType if shouldRate => s"rate(${metricBuilder.name.mkString(metadataScopeSeparator())})" case other => metricBuilder.name.mkString(metadataScopeSeparator()) } @@ -107,9 +114,10 @@ class MetricExpressionHandler(source: MetricSchemaSource = new MetricSchemaSourc val shouldRate = latchParam.exists { value => value == "true" || value == "1" - } && !sourceLatched + } val expressions = filteredSchemas.map { expressionSchema => - expressionSchema.copy(exprQuery = translateToQuery(expressionSchema.expr, shouldRate)) + expressionSchema.copy(exprQuery = + translateToQuery(expressionSchema.expr, shouldRate, sourceLatched)) } newResponse( diff --git a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala index 729e0855..3da33d39 100644 --- a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala @@ -176,27 +176,45 @@ class MetricExpressionHandlerTest extends AnyFunSuite { test("translate expressions - counters") { val latchedResult = - MetricExpressionHandler.translateToQuery(successRateExpression.expr, shouldRate = false) + MetricExpressionHandler.translateToQuery( + successRateExpression.expr, + shouldRate = false, + sourceLatched = false) assert(latchedResult == "multiply(100.0,divide(success,plus(success,failures)))") val unlatchedResult = - MetricExpressionHandler.translateToQuery(successRateExpression.expr, shouldRate = true) + MetricExpressionHandler.translateToQuery( + successRateExpression.expr, + shouldRate = true, + sourceLatched = false) assert( unlatchedResult == "multiply(100.0,divide(rate(success),plus(rate(success),rate(failures))))") } test("translate histogram expressions - latched does not affect result") { val latchedResult = - MetricExpressionHandler.translateToQuery(latencyP99.expr, shouldRate = false) + MetricExpressionHandler.translateToQuery( + latencyP99.expr, + shouldRate = false, + sourceLatched = false) val unLatchedResult = - MetricExpressionHandler.translateToQuery(latencyP99.expr, shouldRate = true) + MetricExpressionHandler.translateToQuery( + latencyP99.expr, + shouldRate = true, + sourceLatched = false) assert(latchedResult == unLatchedResult) } test("translate histogram expressions - components") { val latencyMinExpr = Expression(latencyMb, Left(Expression.Min)) - val min = MetricExpressionHandler.translateToQuery(latencyMinExpr, shouldRate = false) - val p99 = MetricExpressionHandler.translateToQuery(latencyP99.expr, shouldRate = false) + val min = MetricExpressionHandler.translateToQuery( + latencyMinExpr, + shouldRate = false, + sourceLatched = false) + val p99 = MetricExpressionHandler.translateToQuery( + latencyP99.expr, + shouldRate = false, + sourceLatched = false) assert(min == "latency.min") assert(p99 == "latency.p99") } @@ -204,7 +222,10 @@ class MetricExpressionHandlerTest extends AnyFunSuite { test("translate expressions - gauges") { val connMb = MetricBuilder(name = Seq("client", "connections"), metricType = GaugeType, statsReceiver = sr) - val result = MetricExpressionHandler.translateToQuery(Expression(connMb), shouldRate = false) + val result = MetricExpressionHandler.translateToQuery( + Expression(connMb), + shouldRate = false, + sourceLatched = false) assert(result == "client/connections") } From b0ef44cf61a6013e32112045337b952fe322ff74 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Fri, 17 Sep 2021 21:28:38 +0000 Subject: [PATCH 301/409] util-stats: add default labels for histograms Problem Histogram expressions can represent a bunch of metrics: max/min/p99 etc. which means we have to either name them differently or put a label in expression. And using labels would be a better solution in terms of rendering expressions in dashboards. Solution We hold the Histogram bucket information upon creation, fill the label by defualt. JIRA Issues: CSL-11239 Differential Revision: https://phabricator.twitter.biz/D743299 --- .../handler/MetricExpressionHandler.scala | 55 ++++++++-------- .../exp/MetricExpressionHandlerTest.scala | 63 ++++++++++++------- 2 files changed, 67 insertions(+), 51 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala b/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala index 324c6ec4..66bc03bb 100644 --- a/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala @@ -1,22 +1,27 @@ package com.twitter.server.handler import com.twitter.finagle.Service -import com.twitter.finagle.http.{MediaType, Request, Response, Uri} -import com.twitter.finagle.stats.MetricBuilder.{CounterType, CounterishGaugeType} -import com.twitter.finagle.stats.exp.Expression._ -import com.twitter.finagle.stats.exp.{ - ConstantExpression, - Expression, - FunctionExpression, - HistogramExpression, - MetricExpression, - NoExpression -} -import com.twitter.finagle.stats.{MetricBuilder, StatsFormatter, metadataScopeSeparator} +import com.twitter.finagle.http.MediaType +import com.twitter.finagle.http.Request +import com.twitter.finagle.http.Response +import com.twitter.finagle.http.Uri +import com.twitter.finagle.stats.MetricBuilder.CounterType +import com.twitter.finagle.stats.MetricBuilder.CounterishGaugeType +import com.twitter.finagle.stats.exp.ConstantExpression +import com.twitter.finagle.stats.exp.Expression +import com.twitter.finagle.stats.exp.FunctionExpression +import com.twitter.finagle.stats.exp.HistogramExpression +import com.twitter.finagle.stats.exp.MetricExpression +import com.twitter.finagle.stats.exp.NoExpression +import com.twitter.finagle.stats.MetricBuilder +import com.twitter.finagle.stats.StatsFormatter +import com.twitter.finagle.stats.metadataScopeSeparator import com.twitter.io.Buf -import com.twitter.server.handler.MetricExpressionHandler.{Version, translateToQuery} +import com.twitter.server.handler.MetricExpressionHandler.Version +import com.twitter.server.handler.MetricExpressionHandler.translateToQuery import com.twitter.server.util.HttpUtils.newResponse -import com.twitter.server.util.{AdminJsonConverter, MetricSchemaSource} +import com.twitter.server.util.AdminJsonConverter +import com.twitter.server.util.MetricSchemaSource import com.twitter.util.Future object MetricExpressionHandler { @@ -32,14 +37,16 @@ object MetricExpressionHandler { private[server] def translateToQuery( expr: Expression, shouldRate: Boolean, - sourceLatched: Boolean + sourceLatched: Boolean, + labels: Map[String, String] ): String = expr match { - case HistogramExpression(schema, component) => getHisto(schema, component) + case HistogramExpression(schema, _) => getHisto(schema, labels) case MetricExpression(schema) => getMetric(schema, shouldRate, sourceLatched) case ConstantExpression(repr) => repr case FunctionExpression(funcName, exprs) => - s"$funcName(${exprs.map { expr => translateToQuery(expr, shouldRate, sourceLatched) }.mkString(",")})" + s"$funcName(${exprs + .map { expr => translateToQuery(expr, shouldRate, sourceLatched, labels) }.mkString(",")})" case NoExpression => "null" } @@ -48,18 +55,10 @@ object MetricExpressionHandler { // e.g.request_latency.p9999 or request_latency.min private def getHisto( metricBuilder: MetricBuilder, - histoComponent: Either[HistogramComponent, Double] + labels: Map[String, String] ): String = { val name = metricBuilder.name.mkString(metadataScopeSeparator()) - val component = histoComponent match { - case Right(percentile) => statsFormatter.labelPercentile(percentile) - case Left(Min) => statsFormatter.labelMin - case Left(Max) => statsFormatter.labelMax - case Left(Avg) => statsFormatter.labelAverage - case Left(Sum) => statsFormatter.labelSum - case Left(Count) => statsFormatter.labelCount - } - statsFormatter.histoName(name, component) + statsFormatter.histoName(name, labels("bucket")) } // Form metrics other than histograms, rate() for unlatched counters @@ -117,7 +116,7 @@ class MetricExpressionHandler(source: MetricSchemaSource = new MetricSchemaSourc } val expressions = filteredSchemas.map { expressionSchema => expressionSchema.copy(exprQuery = - translateToQuery(expressionSchema.expr, shouldRate, sourceLatched)) + translateToQuery(expressionSchema.expr, shouldRate, sourceLatched, expressionSchema.labels)) } newResponse( diff --git a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala index 3da33d39..b6bd7893 100644 --- a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala @@ -2,18 +2,24 @@ package com.twitter.server.handler.exp import com.twitter.conversions.DurationOps._ import com.twitter.finagle.http.Request -import com.twitter.finagle.stats.MetricBuilder.{CounterType, GaugeType, HistogramType} -import com.twitter.finagle.stats.exp.{ - Expression, - ExpressionSchema, - ExpressionSchemaKey, - GreaterThan, - MonotoneThresholds -} -import com.twitter.finagle.stats.{InMemoryStatsReceiver, MetricBuilder, SchemaRegistry} +import com.twitter.finagle.stats.MetricBuilder.CounterType +import com.twitter.finagle.stats.MetricBuilder.GaugeType +import com.twitter.finagle.stats.MetricBuilder.HistogramType +import com.twitter.finagle.stats.exp.Expression +import com.twitter.finagle.stats.exp.ExpressionSchema +import com.twitter.finagle.stats.exp.ExpressionSchemaKey +import com.twitter.finagle.stats.exp.GreaterThan +import com.twitter.finagle.stats.exp.MonotoneThresholds +import com.twitter.finagle.stats.InMemoryStatsReceiver +import com.twitter.finagle.stats.MetricBuilder +import com.twitter.finagle.stats.SchemaRegistry import com.twitter.server.handler.MetricExpressionHandler -import com.twitter.server.util.{AdminJsonConverter, JsonUtils, MetricSchemaSource} -import com.twitter.util.{Await, Awaitable, Duration} +import com.twitter.server.util.AdminJsonConverter +import com.twitter.server.util.JsonUtils +import com.twitter.server.util.MetricSchemaSource +import com.twitter.util.Await +import com.twitter.util.Awaitable +import com.twitter.util.Duration import org.scalatest.funsuite.AnyFunSuite class MetricExpressionHandlerTest extends AnyFunSuite { @@ -140,7 +146,8 @@ class MetricExpressionHandlerTest extends AnyFunSuite { | "labels" : { | "process_path" : "Unspecified", | "service_name" : "Unspecified", - | "role" : "NoRoleSpecified" + | "role" : "NoRoleSpecified", + | "bucket": "p99" | }, | "namespaces" : ["tenantName"], | "expression" : "latency.p99", @@ -179,14 +186,16 @@ class MetricExpressionHandlerTest extends AnyFunSuite { MetricExpressionHandler.translateToQuery( successRateExpression.expr, shouldRate = false, - sourceLatched = false) + sourceLatched = false, + successRateExpression.labels) assert(latchedResult == "multiply(100.0,divide(success,plus(success,failures)))") val unlatchedResult = MetricExpressionHandler.translateToQuery( successRateExpression.expr, shouldRate = true, - sourceLatched = false) + sourceLatched = false, + successRateExpression.labels) assert( unlatchedResult == "multiply(100.0,divide(rate(success),plus(rate(success),rate(failures))))") } @@ -196,25 +205,30 @@ class MetricExpressionHandlerTest extends AnyFunSuite { MetricExpressionHandler.translateToQuery( latencyP99.expr, shouldRate = false, - sourceLatched = false) + sourceLatched = false, + latencyP99.labels) val unLatchedResult = MetricExpressionHandler.translateToQuery( latencyP99.expr, shouldRate = true, - sourceLatched = false) + sourceLatched = false, + latencyP99.labels) assert(latchedResult == unLatchedResult) } test("translate histogram expressions - components") { - val latencyMinExpr = Expression(latencyMb, Left(Expression.Min)) + + val latencyMin = ExpressionSchema("min", Expression(latencyMb, Left(Expression.Min))) val min = MetricExpressionHandler.translateToQuery( - latencyMinExpr, + latencyMin.expr, shouldRate = false, - sourceLatched = false) + sourceLatched = false, + latencyMin.labels) val p99 = MetricExpressionHandler.translateToQuery( latencyP99.expr, shouldRate = false, - sourceLatched = false) + sourceLatched = false, + latencyP99.labels) assert(min == "latency.min") assert(p99 == "latency.p99") } @@ -225,7 +239,8 @@ class MetricExpressionHandlerTest extends AnyFunSuite { val result = MetricExpressionHandler.translateToQuery( Expression(connMb), shouldRate = false, - sourceLatched = false) + sourceLatched = false, + labels = Map()) assert(result == "client/connections") } @@ -292,7 +307,8 @@ class MetricExpressionHandlerTest extends AnyFunSuite { | "labels" : { | "process_path" : "Unspecified", | "service_name" : "Unspecified", - | "role" : "NoRoleSpecified" + | "role" : "NoRoleSpecified", + | "bucket": "p99" | }, | "namespaces" : ["tenantName"], | "expression" : "latency.p99", @@ -323,7 +339,8 @@ class MetricExpressionHandlerTest extends AnyFunSuite { | "labels" : { | "process_path" : "Unspecified", | "service_name" : "Unspecified", - | "role" : "NoRoleSpecified" + | "role" : "NoRoleSpecified", + | "bucket": "p99" | }, | "namespaces" : ["tenantName"], | "expression" : "latency.p99", From bbc49068417ffdfb750baee05e606d6702eaf451 Mon Sep 17 00:00:00 2001 From: Helen Woldesenbet Date: Tue, 21 Sep 2021 20:16:43 +0000 Subject: [PATCH 302/409] twitter-server: Registry and Toggle Handler updated with ScalaObjectMapper Problem RegistryHandler and ToggleHandler endpoint need to be updated to use util-jackson ScalaObjectMapper APIs Solution/Result used ScalaObjectMapper JIRA Issues: CSL-11306 Differential Revision: https://phabricator.twitter.biz/D743262 --- .../twitter/server/handler/RegistryHandler.scala | 10 +++++++--- .../twitter/server/handler/ToggleHandler.scala | 16 ++++++++++++---- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/handler/RegistryHandler.scala b/server/src/main/scala/com/twitter/server/handler/RegistryHandler.scala index cf52d9e5..de314c7a 100644 --- a/server/src/main/scala/com/twitter/server/handler/RegistryHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/RegistryHandler.scala @@ -1,11 +1,15 @@ package com.twitter.server.handler import com.twitter.finagle.Service -import com.twitter.finagle.http.{Request, Response} +import com.twitter.finagle.http.Request +import com.twitter.finagle.http.Response import com.twitter.server.util.AdminJsonConverter import com.twitter.server.util.HttpUtils.newOk import com.twitter.util.Future -import com.twitter.util.registry.{Formatter, GlobalRegistry, Registry, SimpleRegistry} +import com.twitter.util.registry.Formatter +import com.twitter.util.registry.GlobalRegistry +import com.twitter.util.registry.Registry +import com.twitter.util.registry.SimpleRegistry private object RegistryHandler { sealed trait Matcher { @@ -76,7 +80,7 @@ class RegistryHandler extends Service[Request, Response] { private[handler] def jsonResponse(filter: Option[String]): String = { val filtered: Registry = filterRegistry(filter) val asMap: Map[String, Object] = Formatter.asMap(filtered) - AdminJsonConverter.writeToString(asMap) + AdminJsonConverter.prettyObjectMapper.writeValueAsString(asMap) } } diff --git a/server/src/main/scala/com/twitter/server/handler/ToggleHandler.scala b/server/src/main/scala/com/twitter/server/handler/ToggleHandler.scala index 7c67ae2b..20e31017 100644 --- a/server/src/main/scala/com/twitter/server/handler/ToggleHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/ToggleHandler.scala @@ -1,11 +1,18 @@ package com.twitter.server.handler import com.twitter.finagle.Service -import com.twitter.finagle.http.{Fields, Method, Request, Response, Status} -import com.twitter.finagle.toggle.{StandardToggleMap, Toggle, ToggleMap} +import com.twitter.finagle.http.Fields +import com.twitter.finagle.http.Method +import com.twitter.finagle.http.Request +import com.twitter.finagle.http.Response +import com.twitter.finagle.http.Status +import com.twitter.finagle.toggle.StandardToggleMap +import com.twitter.finagle.toggle.Toggle +import com.twitter.finagle.toggle.ToggleMap import com.twitter.io.Buf import com.twitter.server.util.AdminJsonConverter -import com.twitter.server.util.HttpUtils.{newOk, newResponse} +import com.twitter.server.util.HttpUtils.newOk +import com.twitter.server.util.HttpUtils.newResponse import com.twitter.util.Future import com.twitter.util.logging.Logger import scala.collection.mutable @@ -188,7 +195,8 @@ class ToggleHandler private[handler] (registeredLibrariesFn: () => Map[String, T headers: Iterable[(String, Object)] = Seq.empty ): Future[Response] = { // sort the errors to make the response more deterministic. - val body = AdminJsonConverter.writeToString(GenericResponse(msg, errors.sorted)) + val body = + AdminJsonConverter.prettyObjectMapper.writeValueAsString(GenericResponse(msg, errors.sorted)) newResponse( status = status, contentType = "text/plain;charset=UTF-8", From 7a8e39246f12216026dac062803fea309ac79a0a Mon Sep 17 00:00:00 2001 From: Kris Pineda Date: Wed, 22 Sep 2021 16:35:19 +0000 Subject: [PATCH 303/409] twitter-server: Fix the broken link in Contributing Problem/Solution Fix the broken link in Contributing Signed-off-by: Jing Yan Differential Revision: https://phabricator.twitter.biz/D747739 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 90042271..c4c5dc37 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ public API breaks and changes in runtime behavior. ## Contributing We feel that a welcoming community is important and we ask that you follow Twitter's -[Open Source Code of Conduct](https://github.com/twitter/code-of-conduct/blob/release/code-of-conduct.md) +[Open Source Code of Conduct](https://github.com/twitter/.github/blob/main/code-of-conduct.md) in all interactions with the community. The `release` branch of this repository contains the latest stable release of From 9eba2ee26bba6924e6e3367014e3bbb52e31f36a Mon Sep 17 00:00:00 2001 From: Joy Bestourous Date: Thu, 23 Sep 2021 19:44:58 +0000 Subject: [PATCH 304/409] util-stats: propagate CounterishGaugeType Problem: Regardess of having set `.withCounterishGauge()`, when building a gauge via the `MetricBuilder.gauge()` api, we always return a gauge schema. This makes the `.withCounterishGauge()`, in effect, non-functional. Solution: Rather than copying a `MetricBuilder` and resetting its `metricType`, simply verify that we're only calling the `.gauge()` `.counter()` or `.stat()` api on a `MetricBuilder` with the appropriate `metricType` JIRA Issues: CSL-11280, CSL-11336 Differential Revision: https://phabricator.twitter.biz/D749274 --- .../test/scala/com/twitter/server/AdminHttpServerTest.scala | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala b/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala index 8dbc4c4e..0cc808df 100644 --- a/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala +++ b/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala @@ -1,12 +1,14 @@ package com.twitter.server import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.{Http, ListeningServer} +import com.twitter.finagle.Http +import com.twitter.finagle.ListeningServer import com.twitter.finagle.http._ import com.twitter.server.util.HttpUtils._ import com.twitter.util._ import java.net.InetSocketAddress -import org.scalatest.concurrent.{Eventually, IntegrationPatience} +import org.scalatest.concurrent.Eventually +import org.scalatest.concurrent.IntegrationPatience import org.scalatest.funsuite.AnyFunSuite class MockMetricsExporter extends HttpMuxHandler { From 2784c327e14eec5f689c24e8e7fe3e1bd23a7853 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Thu, 30 Sep 2021 22:31:17 +0000 Subject: [PATCH 305/409] Twitter-oss: Prepare OSS libraries for release 21.9.0 Problem We want to release the next versions of our Twitter OSS libraries 21.9.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D754337 --- CHANGELOG.rst | 3 +++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a62ae870..e491c668 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +21.9.0 +------ + Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/README.md b/README.md index c4c5dc37..320da6d9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index ac29c085..76222518 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.9.0-SNAPSHOT" +val releaseVersion = "21.9.0" val logbackVersion = "1.2.6" val opencensusVersion = "0.19.1" From 35770361fe5cee4819ce9ad941cf3134b6818a16 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Fri, 1 Oct 2021 01:09:58 +0000 Subject: [PATCH 306/409] Twitter-oss: update OSS libraries post-release to 21.10.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 21.10.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D754461 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 320da6d9..c4c5dc37 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 76222518..6e210d0f 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.9.0" +val releaseVersion = "21.10.0-SNAPSHOT" val logbackVersion = "1.2.6" val opencensusVersion = "0.19.1" From 54437e179db0f2273be9148f70c36eb028283934 Mon Sep 17 00:00:00 2001 From: adams Date: Thu, 21 Oct 2021 17:06:53 +0000 Subject: [PATCH 307/409] [twitter-server] Mark Targets Bazel Compatible Problem/Solution Tests are bazel compatible JIRA Issues: DPB-14380 Differential Revision: https://phabricator.twitter.biz/D766988 --- server/src/test/scala/BUILD | 1 + slf4j-jdk14/src/test/scala/BUILD | 1 + 2 files changed, 2 insertions(+) diff --git a/server/src/test/scala/BUILD b/server/src/test/scala/BUILD index 7de6e580..cd1e2ab4 100644 --- a/server/src/test/scala/BUILD +++ b/server/src/test/scala/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", "3rdparty/jvm/com/fasterxml/jackson/module:jackson-module-scala", diff --git a/slf4j-jdk14/src/test/scala/BUILD b/slf4j-jdk14/src/test/scala/BUILD index 3a585299..57357959 100644 --- a/slf4j-jdk14/src/test/scala/BUILD +++ b/slf4j-jdk14/src/test/scala/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", From 71ed97b46f72f5e35d17afd30bc721b44ceb51e4 Mon Sep 17 00:00:00 2001 From: Joy Bestourous Date: Wed, 27 Oct 2021 17:28:34 +0000 Subject: [PATCH 308/409] Twitter-oss: Prepare OSS libraries for release 21.10.0 Problem We want to release the next versions of our Twitter OSS libraries - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-11391 Differential Revision: https://phabricator.twitter.biz/D770597 --- CHANGELOG.rst | 5 +++++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e491c668..4c38852b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +21.10.0 +------- + +No Changes + 21.9.0 ------ diff --git a/README.md b/README.md index c4c5dc37..320da6d9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 6e210d0f..24354d7f 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.10.0-SNAPSHOT" +val releaseVersion = "21.10.0" val logbackVersion = "1.2.6" val opencensusVersion = "0.19.1" From 7398d039f8f52bb99aa9d6fc9a31be83ce3b1459 Mon Sep 17 00:00:00 2001 From: Joy Bestourous Date: Wed, 27 Oct 2021 20:03:33 +0000 Subject: [PATCH 309/409] Twitter-oss: update OSS libraries post-release to 21.11.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 21.11.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. JIRA Issues: CSL-11391 Differential Revision: https://phabricator.twitter.biz/D771421 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 320da6d9..c4c5dc37 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 24354d7f..1232bca0 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.10.0" +val releaseVersion = "21.11.0-SNAPSHOT" val logbackVersion = "1.2.6" val opencensusVersion = "0.19.1" From 7511c43d3242b16c280092ca5f17d5b82b249add Mon Sep 17 00:00:00 2001 From: Joy Bestourous Date: Mon, 1 Nov 2021 20:39:35 +0000 Subject: [PATCH 310/409] twitter-server: expose some handler methods Problem: We'd like to reuse some methods in twitter-server handler classes. Solution: Make scopes less strict in a few files. JIRA Issues: CSL-11346 Differential Revision: https://phabricator.twitter.biz/D765800 --- .../handler/HistogramQueryHandler.scala | 12 ++++++++---- .../server/handler/MetricQueryHandler.scala | 8 ++++++-- .../handler/MetricTypeQueryHandler.scala | 19 ++++++++++++------- 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/handler/HistogramQueryHandler.scala b/server/src/main/scala/com/twitter/server/handler/HistogramQueryHandler.scala index a99910c9..1a224f94 100644 --- a/server/src/main/scala/com/twitter/server/handler/HistogramQueryHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/HistogramQueryHandler.scala @@ -1,8 +1,12 @@ package com.twitter.server.handler import com.twitter.finagle.Service -import com.twitter.finagle.http.{Request, Response, Uri} -import com.twitter.finagle.stats.{BucketAndCount, HistogramDetail, WithHistogramDetails} +import com.twitter.finagle.http.Request +import com.twitter.finagle.http.Response +import com.twitter.finagle.http.Uri +import com.twitter.finagle.stats.BucketAndCount +import com.twitter.finagle.stats.HistogramDetail +import com.twitter.finagle.stats.WithHistogramDetails import com.twitter.io.Buf import com.twitter.server.util.AdminJsonConverter import com.twitter.server.util.HtmlUtils.escapeHtml @@ -14,7 +18,7 @@ object HistogramQueryHandler { private val ContentTypeJson = "application/json;charset=UTF-8" private val ContentTypeHtml = "text/html;charset=UTF-8" - private case class Summary( + private[handler] case class Summary( name: String, count: Long, sum: Long, @@ -215,7 +219,7 @@ private[server] class HistogramQueryHandler(details: WithHistogramDetails) if (bc.upperLimit >= Int.MaxValue) bc.lowerLimit else (bc.upperLimit + bc.lowerLimit) / 2.0 - private[this] def generateSummary(histoName: String): Option[Summary] = { + private[handler] def generateSummary(histoName: String): Option[Summary] = { histograms.get(histoName).map { detail => val bcs = detail.counts.sortBy(_.lowerLimit) diff --git a/server/src/main/scala/com/twitter/server/handler/MetricQueryHandler.scala b/server/src/main/scala/com/twitter/server/handler/MetricQueryHandler.scala index 33a2fae5..2f5fc879 100644 --- a/server/src/main/scala/com/twitter/server/handler/MetricQueryHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/MetricQueryHandler.scala @@ -1,11 +1,15 @@ package com.twitter.server.handler import com.twitter.finagle.Service -import com.twitter.finagle.http.{MediaType, Request, Response, Uri} +import com.twitter.finagle.http.MediaType +import com.twitter.finagle.http.Request +import com.twitter.finagle.http.Response +import com.twitter.finagle.http.Uri import com.twitter.io.Buf import com.twitter.server.util.HtmlUtils.escapeHtml import com.twitter.server.util.HttpUtils.newResponse -import com.twitter.server.util.{AdminJsonConverter, MetricSource} +import com.twitter.server.util.AdminJsonConverter +import com.twitter.server.util.MetricSource import com.twitter.util.Future private object MetricQueryHandler { diff --git a/server/src/main/scala/com/twitter/server/handler/MetricTypeQueryHandler.scala b/server/src/main/scala/com/twitter/server/handler/MetricTypeQueryHandler.scala index e01e302c..f5fa5dfe 100644 --- a/server/src/main/scala/com/twitter/server/handler/MetricTypeQueryHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/MetricTypeQueryHandler.scala @@ -1,11 +1,16 @@ package com.twitter.server.handler import com.twitter.finagle.Service -import com.twitter.finagle.http.{MediaType, Request, Response, Uri} -import com.twitter.finagle.stats.{HistogramDetail, WithHistogramDetails} +import com.twitter.finagle.http.MediaType +import com.twitter.finagle.http.Request +import com.twitter.finagle.http.Response +import com.twitter.finagle.http.Uri +import com.twitter.finagle.stats.HistogramDetail +import com.twitter.finagle.stats.WithHistogramDetails import com.twitter.io.Buf import com.twitter.server.util.HttpUtils.newResponse -import com.twitter.server.util.{AdminJsonConverter, MetricSource} +import com.twitter.server.util.AdminJsonConverter +import com.twitter.server.util.MetricSource import com.twitter.util.Future /** @@ -44,22 +49,22 @@ class MetricTypeQueryHandler( details: Option[WithHistogramDetails] = None) extends Service[Request, Response] { - private[this] def histograms: Map[String, HistogramDetail] = + protected def histograms: Map[String, HistogramDetail] = details match { case Some(histos) => histos.histogramDetails case None => Map.empty } - private[this] def query(keys: Iterable[String]) = + protected def query(keys: Iterable[String]): Iterable[MetricSource.MetricTypeInfo] = for (k <- keys; e <- source.getType(k)) yield e - private[this] def getHistos(): Iterable[MetricSource.MetricTypeInfo] = { + protected def getHistos(): Iterable[MetricSource.MetricTypeInfo] = { histograms.map { case (name, _) => MetricSource.MetricTypeInfo(name, "histogram") } } - private[this] def queryHistos(keys: Set[String]): Iterable[MetricSource.MetricTypeInfo] = { + protected def queryHistos(keys: Set[String]): Iterable[MetricSource.MetricTypeInfo] = { histograms.filterKeys(keys.contains).map { case (name, _) => MetricSource.MetricTypeInfo(name, "histogram") } From 4ba9b7e4377bb7687146d90e6a037c17e5b44f70 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Mon, 8 Nov 2021 15:42:51 +0000 Subject: [PATCH 311/409] 3rdparty: jackson upgrade to 2.13.0 Problem We are on jackson 2.11.x, but we want to upgrade to a new version. Solution Upgrade to 2.13.0 Differential Revision: https://phabricator.twitter.biz/D744627 --- CHANGELOG.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4c38852b..2812597f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Runtime Behavior Changes +~~~~~~~~~~~~~~~~~~~~~~~~ + +* Bump version of Jackson to 2.13.0. ``PHAB_ID=D744627`` + 21.10.0 ------- From 1aa93b50d0052465834f651d0456b70ed335e598 Mon Sep 17 00:00:00 2001 From: Dorothy Ordogh Date: Tue, 9 Nov 2021 01:03:59 +0000 Subject: [PATCH 312/409] finagle|finatra|scrooge|ts: Start every workflow with a clean slate Problem OS builds are failing because of a compile error caused by generated code that has been saved on the file system but the code has been removed in a recent commit. Solution/Result Blowing away the cached files for each workflow has [[https://github.com/twitter/finatra/pull/570 | proven to help ]], so let's do that as an interim solution. JIRA Issues: CSL-11456 Differential Revision: https://phabricator.twitter.biz/D779807 --- .github/workflows/continuous-integration.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index a33e51f6..12290267 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -59,6 +59,7 @@ jobs: no-test: true publish-m2: true verbose: true + clean-files: true - name: test run: ${{ format('./sbt ++{0} clean coverage test coverageReport', matrix.scala) }} - name: coverage From 393428fe737bb93685f6199fc08165558aab4e4e Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Tue, 9 Nov 2021 19:24:30 +0000 Subject: [PATCH 313/409] Revert "3rdparty: jackson upgrade to 2.13.0" Problem 2.13.0 introduced a regression, where they were double-serializing. We're worried that this will lead to a substantial performance regression. Github ticket: https://github.com/FasterXML/jackson-databind/issues/3308 Solution This reverts commit e27353b784543196b9f5fd572110e4b4fe04f4c4. JIRA Issues: CSL-11459 Differential Revision: https://phabricator.twitter.biz/D780287 --- CHANGELOG.rst | 5 ----- 1 file changed, 5 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 2812597f..4c38852b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,11 +7,6 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- -Runtime Behavior Changes -~~~~~~~~~~~~~~~~~~~~~~~~ - -* Bump version of Jackson to 2.13.0. ``PHAB_ID=D744627`` - 21.10.0 ------- From e3b7fbd35e6c5032cf6b2695742bd2bd7bdbbd15 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Wed, 10 Nov 2021 22:01:13 +0000 Subject: [PATCH 314/409] Update PROJECT files Problem/Solution Update PROJECT files. Differential Revision: https://phabricator.twitter.biz/D781463 --- PROJECT.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/PROJECT.yml b/PROJECT.yml index 22b76c28..b2c22cb8 100644 --- a/PROJECT.yml +++ b/PROJECT.yml @@ -2,11 +2,9 @@ owners: - banderson - - ccoco - hallam - ibennett - koliver - - mdannenberg - mnakamura - roanta - yufang From 82677f82488afef915fa499b941fb0c4b805770e Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Thu, 11 Nov 2021 00:11:53 +0000 Subject: [PATCH 315/409] util-stats: Add Stringly Expression constructor Problem It can be troublesome to pass around MetricBuilders or StatsReceiver for users to create expressions. Solution Support creating expression from strings. Result Using string to create expressions could be a bit error-prone, however, the convenience wins the tradeoff comparison here. We added some validation, but it isn't going to guard against all errors since many metrics are created lazily. JIRA Issues: CSL-11283 Differential Revision: https://phabricator.twitter.biz/D777021 --- .../com/twitter/server/handler/MetricExpressionHandler.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala b/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala index 66bc03bb..15e6b928 100644 --- a/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala @@ -15,6 +15,7 @@ import com.twitter.finagle.stats.exp.MetricExpression import com.twitter.finagle.stats.exp.NoExpression import com.twitter.finagle.stats.MetricBuilder import com.twitter.finagle.stats.StatsFormatter +import com.twitter.finagle.stats.exp.StringExpression import com.twitter.finagle.stats.metadataScopeSeparator import com.twitter.io.Buf import com.twitter.server.handler.MetricExpressionHandler.Version @@ -47,6 +48,7 @@ object MetricExpressionHandler { case FunctionExpression(funcName, exprs) => s"$funcName(${exprs .map { expr => translateToQuery(expr, shouldRate, sourceLatched, labels) }.mkString(",")})" + case StringExpression(expr) => expr.mkString(metadataScopeSeparator()) case NoExpression => "null" } From 4f7d874c7ff31adddc062e7d7780dbee30a0be96 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Tue, 16 Nov 2021 21:36:28 +0000 Subject: [PATCH 316/409] finagle-core: Add failures as an metric expression Problem/Solution We would like to have failures as an expression, for client and server, to be auto rendered in the dashboards. JIRA Issues: CSL-11437 Differential Revision: https://phabricator.twitter.biz/D783013 --- .../handler/MetricExpressionHandler.scala | 15 +++++++++---- .../exp/MetricExpressionHandlerTest.scala | 22 +++++++++++++++++-- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala b/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala index 15e6b928..b0885c8c 100644 --- a/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala @@ -28,6 +28,7 @@ import com.twitter.util.Future object MetricExpressionHandler { private val Version = 1.1 private val statsFormatter = StatsFormatter.default + private val Wildcard = "/*" /** * Translate the [[Expression]] object to a single line string which represents generic @@ -43,7 +44,8 @@ object MetricExpressionHandler { ): String = expr match { case HistogramExpression(schema, _) => getHisto(schema, labels) - case MetricExpression(schema) => getMetric(schema, shouldRate, sourceLatched) + case MetricExpression(schema, showRollup) => + getMetric(schema, showRollup, shouldRate, sourceLatched) case ConstantExpression(repr) => repr case FunctionExpression(funcName, exprs) => s"$funcName(${exprs @@ -66,15 +68,20 @@ object MetricExpressionHandler { // Form metrics other than histograms, rate() for unlatched counters private def getMetric( metricBuilder: MetricBuilder, + showRollUp: Boolean, shouldRate: Boolean, sourceLatched: Boolean ): String = { + val metric = metricBuilder.name.mkString(metadataScopeSeparator()) + { + if (showRollUp) Wildcard + else "" + } metricBuilder.metricType match { case CounterType if shouldRate && !sourceLatched => - s"rate(${metricBuilder.name.mkString(metadataScopeSeparator())})" + s"rate($metric)" case CounterishGaugeType if shouldRate => - s"rate(${metricBuilder.name.mkString(metadataScopeSeparator())})" - case other => metricBuilder.name.mkString(metadataScopeSeparator()) + s"rate($metric)" + case _ => metric } } } diff --git a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala index b6bd7893..e726fef4 100644 --- a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala @@ -49,10 +49,14 @@ class MetricExpressionHandlerTest extends AnyFunSuite { val latencyP99 = ExpressionSchema("latency_p99", Expression(latencyMb, Right(0.99))).withNamespace("tenantName") + val failureExpression = + ExpressionSchema("failures", Expression(failuresMb, true)) + val expressionSchemaMap: Map[ExpressionSchemaKey, ExpressionSchema] = Map( ExpressionSchemaKey("success_rate", Map(), Seq()) -> successRateExpression, ExpressionSchemaKey("throughput", Map(), Seq()) -> throughputExpression, - ExpressionSchemaKey("latency", Map(), Seq("path", "to", "tenantName")) -> latencyP99 + ExpressionSchemaKey("latency", Map(), Seq("path", "to", "tenantName")) -> latencyP99, + ExpressionSchemaKey("failures", Map(), Seq()) -> failureExpression ) val expressionRegistry = new SchemaRegistry { @@ -150,7 +154,21 @@ class MetricExpressionHandlerTest extends AnyFunSuite { | "bucket": "p99" | }, | "namespaces" : ["tenantName"], - | "expression" : "latency.p99", + | "expression" : "latency.p99", + | "bounds" : { + | "kind" : "unbounded" + | }, + | "description" : "Unspecified", + | "unit" : "Unspecified" + | }, + | { + | "name" : "failures", + | "labels" : { + | "process_path" : "Unspecified", + | "service_name" : "Unspecified", + | "role" : "NoRoleSpecified" + | }, + | "expression" : "failures/*", | "bounds" : { | "kind" : "unbounded" | }, From 089dc13ca316c8b8e139c66ae299ae321291b619 Mon Sep 17 00:00:00 2001 From: Dorothy Ordogh Date: Tue, 23 Nov 2021 19:00:17 +0000 Subject: [PATCH 317/409] Twitter-oss: Prepare OSS libraries for release 21.11.0 Problem We want to release the next versions of our Twitter OSS libraries 21.11.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-11489 Differential Revision: https://phabricator.twitter.biz/D789826 --- CHANGELOG.rst | 5 +++++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4c38852b..0feebab3 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +21.11.0 +------- + +No Changes + 21.10.0 ------- diff --git a/README.md b/README.md index c4c5dc37..320da6d9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 1232bca0..f38c9dec 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.11.0-SNAPSHOT" +val releaseVersion = "21.11.0" val logbackVersion = "1.2.6" val opencensusVersion = "0.19.1" From 1bed4177175121e2bd629b5b7aa6c4e1350efb13 Mon Sep 17 00:00:00 2001 From: Dorothy Ordogh Date: Tue, 23 Nov 2021 20:19:35 +0000 Subject: [PATCH 318/409] Twitter-oss: update OSS libraries post-release to 21.12.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 21.12.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 320da6d9..c4c5dc37 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index f38c9dec..e1810397 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.11.0" +val releaseVersion = "21.12.0-SNAPSHOT" val logbackVersion = "1.2.6" val opencensusVersion = "0.19.1" From 0c8ff5f697a4cd15e589a40c76a5b963869c51b6 Mon Sep 17 00:00:00 2001 From: Dorothy Ordogh Date: Tue, 30 Nov 2021 21:08:04 +0000 Subject: [PATCH 319/409] Revert "finagle|finatra|scrooge|ts: Start every workflow with a clean slate" This reverts commit f249362805a90bfad39a8f1f198b6e657fc21190. Differential Revision: https://phabricator.twitter.biz/D791818 --- .github/workflows/continuous-integration.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 12290267..a33e51f6 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -59,7 +59,6 @@ jobs: no-test: true publish-m2: true verbose: true - clean-files: true - name: test run: ${{ format('./sbt ++{0} clean coverage test coverageReport', matrix.scala) }} - name: coverage From 863c341ac9498894ce41812868dce24d011b7b28 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Fri, 3 Dec 2021 18:11:29 +0000 Subject: [PATCH 320/409] csl: Bump SBT to 1.5.5 Problem / Solution Bump SBT to 1.5.5 for CSL libraries and upgrade selected plugins. JIRA Issues: CSL-11423 Differential Revision: https://phabricator.twitter.biz/D792862 --- .github/workflows/continuous-integration.yml | 2 +- project/build.properties | 2 +- project/plugins.sbt | 6 +++--- sbt | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index a33e51f6..85df6cb7 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -3,7 +3,7 @@ name: continuous integration env: JAVA_OPTS: "-Dsbt.log.noformat=true" TRAVIS: "true" # pretend we're TravisCI - SBT_VERSION: 1.5.3 + SBT_VERSION: 1.5.5 defaults: run: diff --git a/project/build.properties b/project/build.properties index 67d27a1d..10fd9eee 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.5.3 +sbt.version=1.5.5 diff --git a/project/plugins.sbt b/project/plugins.sbt index 08dab906..e55bc980 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -3,6 +3,6 @@ resolvers ++= Seq( Resolver.sonatypeRepo("snapshots") ) -addSbtPlugin("com.typesafe.sbt" % "sbt-site" % "1.3.0") -addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.6.1") -addSbtPlugin("com.eed3si9n" % "sbt-unidoc" % "0.4.1") +addSbtPlugin("com.typesafe.sbt" % "sbt-site" % "1.4.1") +addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.9.1") +addSbtPlugin("com.eed3si9n" % "sbt-unidoc" % "0.4.3") diff --git a/sbt b/sbt index 2441ede7..dd557c21 100755 --- a/sbt +++ b/sbt @@ -2,9 +2,9 @@ set -eo pipefail -sbtver="1.5.3" +sbtver="1.5.5" sbtjar="sbt-launch-$sbtver.jar" -sbtsha128="1ca2d0ee419a1f82f512f2aa8556d6e262b37961" +sbtsha128="3beb23e861de5021bd80ca52113b075321580265" sbtrepo="https://repo1.maven.org/maven2/org/scala-sbt/sbt-launch" From b8ff59c83917e901d1ff4a14041e79376a541f58 Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Fri, 3 Dec 2021 21:14:38 +0000 Subject: [PATCH 321/409] finagle|finatra|scrooge|ts|util: Add build version to GH Actions build ccache key Problem Some generated files are cached in the build cache but the thrift files have since been removed, which causes build failures. Solution Add a way to easily invalidate the build cache so if we experience something like this again, we can simply bump the version number and move on to a successful build. Differential Revision: https://phabricator.twitter.biz/D792859 --- .github/workflows/continuous-integration.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 85df6cb7..d9f5c804 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -4,6 +4,7 @@ env: JAVA_OPTS: "-Dsbt.log.noformat=true" TRAVIS: "true" # pretend we're TravisCI SBT_VERSION: 1.5.5 + BUILD_VERSION: "v1" # bump this if builds are failing due to a bad cache defaults: run: @@ -42,7 +43,7 @@ jobs: ~/.ivy2/cache ~/.ivy2/local/com.twitter ~/.sbt - key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ env.SBT_VERSION }}-${{ matrix.scala }}-${{ matrix.java }} + key: ${{ runner.os }}-build-${{ env.BUILD_VERSION }}-${{ env.cache-name }}-${{ env.SBT_VERSION }}-${{ matrix.scala }}-${{ matrix.java }} - name: update cache run: | if [ -f ~/.ivy2/cache ]; then From 9ae36df2a3f2f144a38149a1159a7a94080fbd1b Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Wed, 15 Dec 2021 05:58:36 +0000 Subject: [PATCH 322/409] [3rdparty] logback 1.2.8 This fixes remote code execution vulnerability in logback, similar to log4shell. See http://logback.qos.ch/news.html Differential Revision: https://phabricator.twitter.biz/D801083 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index e1810397..6cc4ad0d 100644 --- a/build.sbt +++ b/build.sbt @@ -6,7 +6,7 @@ Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch val releaseVersion = "21.12.0-SNAPSHOT" -val logbackVersion = "1.2.6" +val logbackVersion = "1.2.8" val opencensusVersion = "0.19.1" val slf4jVersion = "1.7.30" From 2f612c3e5bf2565b75ed193ef8191e16c5c00dd1 Mon Sep 17 00:00:00 2001 From: Attila Szegedi Date: Wed, 15 Dec 2021 20:17:00 +0000 Subject: [PATCH 323/409] Twitter-oss: Prepare OSS libraries for release 21.12.00 Problem We want to release the next versions of our Twitter OSS libraries 21.12.00 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D802007 --- CHANGELOG.rst | 5 +++++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0feebab3..f78d6459 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +21.12.0 +------- + +No Changes + 21.11.0 ------- diff --git a/README.md b/README.md index c4c5dc37..320da6d9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 6cc4ad0d..3bb563f4 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.12.0-SNAPSHOT" +val releaseVersion = "21.12.0" val logbackVersion = "1.2.8" val opencensusVersion = "0.19.1" From e96d53dcded33ad6ee84d26da568e981677b5b2b Mon Sep 17 00:00:00 2001 From: Attila Szegedi Date: Wed, 15 Dec 2021 21:30:00 +0000 Subject: [PATCH 324/409] Twitter-oss: update OSS libraries post-release to 22.1.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 22.1.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D802418 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 320da6d9..c4c5dc37 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 3bb563f4..1735950e 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.12.0" +val releaseVersion = "22.1.0-SNAPSHOT" val logbackVersion = "1.2.8" val opencensusVersion = "0.19.1" From 0e0aa158b0984c22d3654345c1c15b0b0abd246e Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Thu, 16 Dec 2021 19:01:18 +0000 Subject: [PATCH 325/409] csl: Bye Kevin Differential Revision: https://phabricator.twitter.biz/D802469 --- PROJECT.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/PROJECT.yml b/PROJECT.yml index b2c22cb8..81c55540 100644 --- a/PROJECT.yml +++ b/PROJECT.yml @@ -4,7 +4,6 @@ owners: - banderson - hallam - ibennett - - koliver - mnakamura - roanta - yufang From da33afe983a21d7e7a9b9a3fe27ecf358ec84d03 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Wed, 12 Jan 2022 18:43:58 +0000 Subject: [PATCH 326/409] 3rdparty: Revert "Revert "3rdparty: jackson upgrade to 2.13.0"" & Upgrade to 2.13.1 Problem We're on an old version of jackson, and we'd like to upgrade to a new version. We tried in the past to upgrade to 2.13.0, but it introduced a regression, where they were double-serializing. We believe 2.13.1 no longer has that regression, so it should be safe to upgrade. Solution This reverts commit e27353b784543196b9f5fd572110e4b4fe04f4c4. JIRA Issues: CSL-11573 Differential Revision: https://phabricator.twitter.biz/D808049 --- CHANGELOG.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f78d6459..f81977ea 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Runtime Behavior Changes +~~~~~~~~~~~~~~~~~~~~~~~~ + +* Bump version of Jackson to 2.13.1. ``PHAB_ID=D808049`` + 21.12.0 ------- From cbc7383760766fe1b4e701b17e7f645b31de3086 Mon Sep 17 00:00:00 2001 From: Gary Grossman Date: Wed, 12 Jan 2022 21:20:38 +0000 Subject: [PATCH 327/409] {util,scrooge,twitter-server,finagle,finatra}: Upgrade sbt to 1.6.1 Problem / Solution Update to sbt 1.6.1 Signed-off-by: Moses Nakamura JIRA Issues: CSL-11546 Differential Revision: https://phabricator.twitter.biz/D810355 --- .github/workflows/continuous-integration.yml | 2 +- project/build.properties | 2 +- sbt | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index d9f5c804..1647a6ea 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -3,7 +3,7 @@ name: continuous integration env: JAVA_OPTS: "-Dsbt.log.noformat=true" TRAVIS: "true" # pretend we're TravisCI - SBT_VERSION: 1.5.5 + SBT_VERSION: 1.6.1 BUILD_VERSION: "v1" # bump this if builds are failing due to a bad cache defaults: diff --git a/project/build.properties b/project/build.properties index 10fd9eee..3161d214 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.5.5 +sbt.version=1.6.1 diff --git a/sbt b/sbt index dd557c21..1d62b302 100755 --- a/sbt +++ b/sbt @@ -2,9 +2,9 @@ set -eo pipefail -sbtver="1.5.5" +sbtver="1.6.1" sbtjar="sbt-launch-$sbtver.jar" -sbtsha128="3beb23e861de5021bd80ca52113b075321580265" +sbtsha128="b69ab7f114d09d859b5e349acdb05d32c9d49072" sbtrepo="https://repo1.maven.org/maven2/org/scala-sbt/sbt-launch" From 2e213f7b1b08ba50c1012c24e5fc47aa178e9ed9 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Wed, 12 Jan 2022 22:28:00 +0000 Subject: [PATCH 328/409] twitter-server/slf4j-log4j12: Mark deprecated Problem Log4j 1.x was EOL in 2015. It is time to drop support for the logging framework from TwitterServer. Solution Mark the BUILD targets and the LoggingHandler as deprecated with the intention to remove. Users are encouraged to use Logback as a replacement. Differential Revision: https://phabricator.twitter.biz/D812006 --- slf4j-log4j12/README.md | 4 ++-- slf4j-log4j12/src/main/resources/BUILD | 5 ++++- slf4j-log4j12/src/main/scala/BUILD | 1 + .../server/handler/slf4j/log4j12/LoggingHandler.scala | 5 ++++- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/slf4j-log4j12/README.md b/slf4j-log4j12/README.md index ad2819f4..41955cc7 100644 --- a/slf4j-log4j12/README.md +++ b/slf4j-log4j12/README.md @@ -1,5 +1,5 @@ -TwitterServer `slf4j-log4j12` -============================ +[DEPRECATED] TwitterServer `slf4j-log4j12` +========================================== This library provides a wrapper over the [`slf4j-log4j12`](https://www.slf4j.org/manual.html#swapping) logging implementation for use in `TwitterServer` along with the appropriate [SLF4J bridges](https://www.slf4j.org/legacy.html) diff --git a/slf4j-log4j12/src/main/resources/BUILD b/slf4j-log4j12/src/main/resources/BUILD index 07015f79..0828ba5c 100644 --- a/slf4j-log4j12/src/main/resources/BUILD +++ b/slf4j-log4j12/src/main/resources/BUILD @@ -4,5 +4,8 @@ resources( "!BUILD*", "**/*", ], - tags = ["bazel-compatible"], + tags = [ + "bazel-compatible", + "deprecated", # Users are encouraged to use twitter-server/logback-classic instead + ], ) diff --git a/slf4j-log4j12/src/main/scala/BUILD b/slf4j-log4j12/src/main/scala/BUILD index 8a344a78..5750ad3d 100644 --- a/slf4j-log4j12/src/main/scala/BUILD +++ b/slf4j-log4j12/src/main/scala/BUILD @@ -9,6 +9,7 @@ scala_library( scope = "forced", # Exposes a service-loaded class. tags = [ "bazel-compatible", + "deprecated", # Users are encouraged to use twitter-server/logback-classic instead "logging_impl_check_whitelisted_target", ], dependencies = [ diff --git a/slf4j-log4j12/src/main/scala/com/twitter/server/handler/slf4j/log4j12/LoggingHandler.scala b/slf4j-log4j12/src/main/scala/com/twitter/server/handler/slf4j/log4j12/LoggingHandler.scala index c6c1e790..b14e6958 100644 --- a/slf4j-log4j12/src/main/scala/com/twitter/server/handler/slf4j/log4j12/LoggingHandler.scala +++ b/slf4j-log4j12/src/main/scala/com/twitter/server/handler/slf4j/log4j12/LoggingHandler.scala @@ -8,9 +8,12 @@ import com.twitter.server.util.HttpUtils._ import com.twitter.util.Future import com.twitter.util.logging.Logging import java.net.URLEncoder -import org.apache.log4j.{Level, LogManager, Logger} +import org.apache.log4j.Level +import org.apache.log4j.LogManager +import org.apache.log4j.Logger import scala.collection.JavaConverters._ +@deprecated("Users are encouraged to use twitter-server/logback-classic.", "2022-01-12") private class LoggingHandler extends com.twitter.server.handler.LoggingHandler with Logging { /** Implementation name */ From 4fdee900c81be7084260d212dfcfe3a9157c7d91 Mon Sep 17 00:00:00 2001 From: Kyle Bahr Date: Wed, 12 Jan 2022 23:01:18 +0000 Subject: [PATCH 329/409] Twitter-oss: Prepare OSS libraries for release 22.1.0 Problem We want to release the next versions of our Twitter OSS libraries 22.1.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-11595 Differential Revision: https://phabricator.twitter.biz/D812047 --- CHANGELOG.rst | 3 +++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f81977ea..fefad04d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +22.1.0 +------ + Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/README.md b/README.md index c4c5dc37..320da6d9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 1735950e..cdfe4339 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "22.1.0-SNAPSHOT" +val releaseVersion = "22.1.0" val logbackVersion = "1.2.8" val opencensusVersion = "0.19.1" From 3f46f2fb08ea62022a042209707a333f77a297ec Mon Sep 17 00:00:00 2001 From: Kyle Bahr Date: Thu, 13 Jan 2022 19:56:30 +0000 Subject: [PATCH 330/409] Twitter-oss: update OSS libraries post-release to 22.2.0 Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 22.2.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. JIRA Issues: CSL-11595 Differential Revision: https://phabricator.twitter.biz/D812865 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 320da6d9..c4c5dc37 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index cdfe4339..144cebc5 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "22.1.0" +val releaseVersion = "22.2.0-SNAPSHOT" val logbackVersion = "1.2.8" val opencensusVersion = "0.19.1" From 6a89c1486daea952f41a7c9c04b56a8602a39c66 Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Fri, 21 Jan 2022 20:04:32 +0000 Subject: [PATCH 331/409] twitter-server: typo and link fix on admin endpoints section Problem/Solution Fix a typo and a link in Twitter Server doc. Differential Revision: https://phabricator.twitter.biz/D817747 --- doc/src/sphinx/Admin.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/src/sphinx/Admin.rst b/doc/src/sphinx/Admin.rst index 04863f3d..d3e0bf9a 100644 --- a/doc/src/sphinx/Admin.rst +++ b/doc/src/sphinx/Admin.rst @@ -383,7 +383,7 @@ See the :ref:`metrics ` section for more information. ~~~~~~~~~~~~~~~~~~~~~~~~~~~ Export metadata information about metrics in json. By default, this exports metadata for every metric. An optional argument `name` can be added to query for a single metric's metadata. Multiple names may be passed in using that argument multiple times. -NB: for a histogram, it may be looked up with our without its metrics.json suffix (.sum, .p95, .max, etc). +NB: for a histogram, it may be looked up with or without its metrics.json suffix (.sum, .p95, .max, etc). :: @@ -476,7 +476,7 @@ Export metadata information about metric expressions in json. These metadata rep :: - /admin/metric/expressions.json?name=http/requests&name=strato_rpc_client/latency_ms.p95 + /admin/metric/expressions.json?name=success_rate :: From b4be9ff71f273a007f880df6ea433f238c2d1f18 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Mon, 7 Feb 2022 20:29:00 +0000 Subject: [PATCH 332/409] util-slf4j-bridge: Add Slf4jBridge trait for handler installation in class construction Problem Installation of the SLF4JBridgeHandler should happen as early as possible, ideally during the construction of the main application. Solution Add a mixin which can be composed into extensions of `c.t.app.App` which will attempt installation of the bridge handler via the `Slf4jBridgeUtility` as part of class construction. Move TwitterServer to use the mixin in util-slf4j-jul-bridge. Result A simple way for extensions of `c.t.app.App` to ensure installation of the SLF4JBridgeHandler is attempted during the constructor of the `App`. Differential Revision: https://phabricator.twitter.biz/D827493 --- server/src/main/scala/BUILD | 1 + .../main/scala/com/twitter/server/Slf4jBridge.scala | 10 ---------- .../main/scala/com/twitter/server/TwitterServer.scala | 1 + 3 files changed, 2 insertions(+), 10 deletions(-) delete mode 100644 server/src/main/scala/com/twitter/server/Slf4jBridge.scala diff --git a/server/src/main/scala/BUILD b/server/src/main/scala/BUILD index ffba4f0b..76b446b5 100644 --- a/server/src/main/scala/BUILD +++ b/server/src/main/scala/BUILD @@ -41,5 +41,6 @@ scala_library( "util/util-jackson/src/main/scala/com/twitter/util/jackson", "util/util-lint/src/main/scala", "util/util-slf4j-api/src/main/scala", + "util/util-slf4j-jul-bridge/src/main/scala", ], ) diff --git a/server/src/main/scala/com/twitter/server/Slf4jBridge.scala b/server/src/main/scala/com/twitter/server/Slf4jBridge.scala deleted file mode 100644 index f5a96273..00000000 --- a/server/src/main/scala/com/twitter/server/Slf4jBridge.scala +++ /dev/null @@ -1,10 +0,0 @@ -package com.twitter.server - -import com.twitter.app.App -import com.twitter.util.logging.Slf4jBridgeUtility - -private[server] trait Slf4jBridge { self: App => - - /** Attempt Slf4jBridgeHandler installation */ - Slf4jBridgeUtility.attemptSlf4jBridgeHandlerInstallation() -} diff --git a/server/src/main/scala/com/twitter/server/TwitterServer.scala b/server/src/main/scala/com/twitter/server/TwitterServer.scala index f9724ffa..c1a00d8f 100644 --- a/server/src/main/scala/com/twitter/server/TwitterServer.scala +++ b/server/src/main/scala/com/twitter/server/TwitterServer.scala @@ -7,6 +7,7 @@ import com.twitter.finagle.DtabFlags import com.twitter.finagle.util.DefaultTimer import com.twitter.util.Timer import com.twitter.util.logging.Logging +import com.twitter.util.logging.Slf4jBridge /** * Twitter Server defines a template from which servers at Twitter are built. From c38a02062922c6bb64860a5e098e4c155ddb5ab0 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Tue, 22 Feb 2022 20:21:49 +0000 Subject: [PATCH 333/409] {util,twitter-server}: Make StringExpression higher fidelity Problem StringExpression is currently incompatible with rating because we don't know whether it's backed by a counter or not! Solution Add a boolean as a quick hack to let us know whether the metric is a counter or not. This API may change rapidly as we learn more about how it should be used. JIRA Issues: CSL-11715 Differential Revision: https://phabricator.twitter.biz/D832571 --- .../handler/MetricExpressionHandler.scala | 4 +- .../exp/MetricExpressionHandlerTest.scala | 254 +++++++++++------- 2 files changed, 167 insertions(+), 91 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala b/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala index b0885c8c..e5d91bef 100644 --- a/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/MetricExpressionHandler.scala @@ -50,7 +50,9 @@ object MetricExpressionHandler { case FunctionExpression(funcName, exprs) => s"$funcName(${exprs .map { expr => translateToQuery(expr, shouldRate, sourceLatched, labels) }.mkString(",")})" - case StringExpression(expr) => expr.mkString(metadataScopeSeparator()) + case StringExpression(expr, isCounter) => + val metric = expr.mkString(metadataScopeSeparator()) + if (isCounter && shouldRate && !sourceLatched) s"rate($metric)" else metric case NoExpression => "null" } diff --git a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala index e726fef4..853e1992 100644 --- a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala @@ -52,11 +52,18 @@ class MetricExpressionHandlerTest extends AnyFunSuite { val failureExpression = ExpressionSchema("failures", Expression(failuresMb, true)) + val uptimeExpression = + ExpressionSchema("uptime", Expression(Seq("jvm", "uptime"), isCounter = true)) + val pendingExpression = + ExpressionSchema("pending", Expression(Seq("srv", "pending"), isCounter = false)) + val expressionSchemaMap: Map[ExpressionSchemaKey, ExpressionSchema] = Map( ExpressionSchemaKey("success_rate", Map(), Seq()) -> successRateExpression, ExpressionSchemaKey("throughput", Map(), Seq()) -> throughputExpression, ExpressionSchemaKey("latency", Map(), Seq("path", "to", "tenantName")) -> latencyP99, - ExpressionSchemaKey("failures", Map(), Seq()) -> failureExpression + ExpressionSchemaKey("failures", Map(), Seq()) -> failureExpression, + ExpressionSchemaKey("uptime", Map(), Seq()) -> uptimeExpression, + ExpressionSchemaKey("pending", Map(), Seq()) -> pendingExpression ) val expressionRegistry = new SchemaRegistry { @@ -106,77 +113,111 @@ class MetricExpressionHandlerTest extends AnyFunSuite { test("Get all expressions") { val expectedResponse = """ - |{ - | "@version" : 1.1, - | "counters_latched" : false, - | "separator_char" : "/", - | "expressions" : [ - | { - | "name" : "success_rate", - | "labels" : { - | "process_path" : "Unspecified", - | "service_name" : "Unspecified", - | "role" : "NoRoleSpecified" - | }, - | "expression" : "multiply(100.0,divide(success,plus(success,failures)))", - | "bounds" : { - | "kind" : "monotone", - | "operator" : ">", - | "bad_threshold" : 99.5, - | "good_threshold" : 99.97, - | "lower_bound_inclusive" : null, - | "upper_bound_exclusive" : null - | }, - | "description" : "Unspecified", - | "unit" : "Unspecified" - | }, - | { - | "name" : "throughput", - | "labels" : { - | "process_path" : "Unspecified", - | "service_name" : "Unspecified", - | "role" : "NoRoleSpecified" - | }, - | "namespaces" : ["path","to","tenantName"], - | "expression" : "plus(success,failures)", - | "bounds" : { - | "kind" : "unbounded" - | }, - | "description" : "Unspecified", - | "unit" : "Unspecified" - | }, - | { - | "name" : "latency_p99", - | "labels" : { - | "process_path" : "Unspecified", - | "service_name" : "Unspecified", - | "role" : "NoRoleSpecified", - | "bucket": "p99" - | }, - | "namespaces" : ["tenantName"], - | "expression" : "latency.p99", - | "bounds" : { - | "kind" : "unbounded" - | }, - | "description" : "Unspecified", - | "unit" : "Unspecified" - | }, - | { - | "name" : "failures", - | "labels" : { - | "process_path" : "Unspecified", - | "service_name" : "Unspecified", - | "role" : "NoRoleSpecified" - | }, - | "expression" : "failures/*", - | "bounds" : { - | "kind" : "unbounded" - | }, - | "description" : "Unspecified", - | "unit" : "Unspecified" - | } - | ] - |}""".stripMargin + |{ + | "@version": 1.1, + | "counters_latched": false, + | "expressions": [ + | { + | "bounds": { + | "kind": "unbounded" + | }, + | "description": "Unspecified", + | "expression": "latency.p99", + | "labels": { + | "bucket": "p99", + | "process_path": "Unspecified", + | "role": "NoRoleSpecified", + | "service_name": "Unspecified" + | }, + | "name": "latency_p99", + | "namespaces": [ + | "tenantName" + | ], + | "unit": "Unspecified" + | }, + | { + | "bounds": { + | "kind": "unbounded" + | }, + | "description": "Unspecified", + | "expression": "srv/pending", + | "labels": { + | "process_path": "Unspecified", + | "role": "NoRoleSpecified", + | "service_name": "Unspecified" + | }, + | "name": "pending", + | "unit": "Unspecified" + | }, + | { + | "bounds": { + | "kind": "unbounded" + | }, + | "description": "Unspecified", + | "expression": "plus(success,failures)", + | "labels": { + | "process_path": "Unspecified", + | "role": "NoRoleSpecified", + | "service_name": "Unspecified" + | }, + | "name": "throughput", + | "namespaces": [ + | "path", + | "to", + | "tenantName" + | ], + | "unit": "Unspecified" + | }, + | { + | "bounds": { + | "kind": "unbounded" + | }, + | "description": "Unspecified", + | "expression": "failures/*", + | "labels": { + | "process_path": "Unspecified", + | "role": "NoRoleSpecified", + | "service_name": "Unspecified" + | }, + | "name": "failures", + | "unit": "Unspecified" + | }, + | { + | "bounds": { + | "kind": "unbounded" + | }, + | "description": "Unspecified", + | "expression": "jvm/uptime", + | "labels": { + | "process_path": "Unspecified", + | "role": "NoRoleSpecified", + | "service_name": "Unspecified" + | }, + | "name": "uptime", + | "unit": "Unspecified" + | }, + | { + | "bounds": { + | "bad_threshold": 99.5, + | "good_threshold": 99.97, + | "kind": "monotone", + | "lower_bound_inclusive": null, + | "operator": ">", + | "upper_bound_exclusive": null + | }, + | "description": "Unspecified", + | "expression": "multiply(100.0,divide(success,plus(success,failures)))", + | "labels": { + | "process_path": "Unspecified", + | "role": "NoRoleSpecified", + | "service_name": "Unspecified" + | }, + | "name": "success_rate", + | "unit": "Unspecified" + | } + | ], + | "separator_char": "/" + |}""".stripMargin JsonUtils.assertJsonResponse( await(expressionHandler(testRequest)).contentString, @@ -200,38 +241,37 @@ class MetricExpressionHandlerTest extends AnyFunSuite { } test("translate expressions - counters") { - val latchedResult = + val unratedStyle = MetricExpressionHandler.translateToQuery( successRateExpression.expr, shouldRate = false, sourceLatched = false, successRateExpression.labels) - assert(latchedResult == "multiply(100.0,divide(success,plus(success,failures)))") + assert(unratedStyle == "multiply(100.0,divide(success,plus(success,failures)))") - val unlatchedResult = + val ratedStyle = MetricExpressionHandler.translateToQuery( successRateExpression.expr, shouldRate = true, sourceLatched = false, successRateExpression.labels) - assert( - unlatchedResult == "multiply(100.0,divide(rate(success),plus(rate(success),rate(failures))))") + assert(ratedStyle == "multiply(100.0,divide(rate(success),plus(rate(success),rate(failures))))") } test("translate histogram expressions - latched does not affect result") { - val latchedResult = + val unratedStyle = MetricExpressionHandler.translateToQuery( latencyP99.expr, shouldRate = false, sourceLatched = false, latencyP99.labels) - val unLatchedResult = + val ratedStyle = MetricExpressionHandler.translateToQuery( latencyP99.expr, shouldRate = true, sourceLatched = false, latencyP99.labels) - assert(latchedResult == unLatchedResult) + assert(ratedStyle == ratedStyle) } test("translate histogram expressions - components") { @@ -262,6 +302,40 @@ class MetricExpressionHandlerTest extends AnyFunSuite { assert(result == "client/connections") } + test("translate expressions - strings") { + val unratedStyle = + MetricExpressionHandler.translateToQuery( + pendingExpression.expr, + shouldRate = false, + sourceLatched = false, + pendingExpression.labels) + val ratedStyle = + MetricExpressionHandler.translateToQuery( + pendingExpression.expr, + shouldRate = true, + sourceLatched = false, + pendingExpression.labels) + assert(ratedStyle == unratedStyle) + } + + test("translate expressions - counter-style strings") { + val unratedStyle = + MetricExpressionHandler.translateToQuery( + uptimeExpression.expr, + shouldRate = false, + sourceLatched = false, + uptimeExpression.labels) + assert(unratedStyle == "jvm/uptime") + + val ratedStyle = + MetricExpressionHandler.translateToQuery( + uptimeExpression.expr, + shouldRate = true, + sourceLatched = false, + uptimeExpression.labels) + assert(ratedStyle == "rate(jvm/uptime)") + } + test("Get expression with name param filters to expressions with that name") { val expectedResponse = """ @@ -306,14 +380,15 @@ class MetricExpressionHandlerTest extends AnyFunSuite { | "separator_char" : "/", | "expressions" : [ | { - | "name" : "throughput", + | "name" : "latency_p99", | "labels" : { | "process_path" : "Unspecified", | "service_name" : "Unspecified", - | "role" : "NoRoleSpecified" + | "role" : "NoRoleSpecified", + | "bucket": "p99" | }, - | "namespaces" : ["path","to","tenantName"], - | "expression" : "plus(success,failures)", + | "namespaces" : ["tenantName"], + | "expression" : "latency.p99", | "bounds" : { | "kind" : "unbounded" | }, @@ -321,21 +396,20 @@ class MetricExpressionHandlerTest extends AnyFunSuite { | "unit" : "Unspecified" | }, | { - | "name" : "latency_p99", + | "name" : "throughput", | "labels" : { | "process_path" : "Unspecified", | "service_name" : "Unspecified", - | "role" : "NoRoleSpecified", - | "bucket": "p99" + | "role" : "NoRoleSpecified" | }, - | "namespaces" : ["tenantName"], - | "expression" : "latency.p99", + | "namespaces" : ["path","to","tenantName"], + | "expression" : "plus(success,failures)", | "bounds" : { | "kind" : "unbounded" | }, | "description" : "Unspecified", | "unit" : "Unspecified" - | } + | } | ] |}""".stripMargin From 4f49d6b8f063367b0c1625b02318f3c2c3056493 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Fri, 25 Feb 2022 14:23:01 +0000 Subject: [PATCH 334/409] Twitter-oss: Prepare OSS libraries for release 22.2.0 Problem We want to release the next versions of our Twitter OSS libraries 22.2.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-11693 Differential Revision: https://phabricator.twitter.biz/D840164 --- CHANGELOG.rst | 5 +++++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index fefad04d..5db97add 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +22.2.0 +------ + +No Changes + 22.1.0 ------ diff --git a/README.md b/README.md index c4c5dc37..320da6d9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 144cebc5..342409af 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "22.2.0-SNAPSHOT" +val releaseVersion = "22.2.0" val logbackVersion = "1.2.8" val opencensusVersion = "0.19.1" From 55cbbdf698c9e5bdbae951a41c0595d4cdee4793 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Fri, 25 Feb 2022 16:40:28 +0000 Subject: [PATCH 335/409] Twitter-oss: update OSS libraries post-release to 22.3.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 22.3.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. JIRA Issues: CSL-11693 Differential Revision: https://phabricator.twitter.biz/D840845 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 320da6d9..c4c5dc37 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 342409af..fe3117aa 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "22.2.0" +val releaseVersion = "22.3.0-SNAPSHOT" val logbackVersion = "1.2.8" val opencensusVersion = "0.19.1" From bda1e5a2b94518b4cc9b5a408f08c1ad455ddab4 Mon Sep 17 00:00:00 2001 From: Lily He Date: Wed, 2 Mar 2022 17:38:44 +0000 Subject: [PATCH 336/409] finagle-stats-core: Export counters in Prometheus format Problem: We want to export Finagle counters in Prometheus format. This format allows Finagle to have rich (dimensional) metrics where metrics have labels attached to them. Solution: Encode counters name, value and their labels in the Prometheus format. Optionally write metadata of the type of metric and the units that the metric counts in. Write the metric to a StringBuilder. JIRA Issues: CSL-11562 Differential Revision: https://phabricator.twitter.biz/D839136 --- .../server/util/MetricSchemaSerializer.scala | 20 +++++-------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala b/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala index a59161f2..f40e889f 100644 --- a/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala +++ b/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala @@ -3,14 +3,10 @@ package com.twitter.server.util import com.fasterxml.jackson.core.JsonGenerator import com.fasterxml.jackson.databind.SerializerProvider import com.fasterxml.jackson.databind.ser.std.StdSerializer -import com.twitter.finagle.stats.MetricBuilder.{ - CounterType, - CounterishGaugeType, - GaugeType, - HistogramType, - UnlatchedCounter -} -import com.twitter.finagle.stats.{MetricBuilder, StatsFormatter, metadataScopeSeparator} +import com.twitter.finagle.stats.MetricBuilder.HistogramType +import com.twitter.finagle.stats.MetricBuilder +import com.twitter.finagle.stats.StatsFormatter +import com.twitter.finagle.stats.metadataScopeSeparator object SchemaSerializer extends StdSerializer[MetricBuilder](classOf[MetricBuilder]) { @@ -44,13 +40,7 @@ object SchemaSerializer extends StdSerializer[MetricBuilder](classOf[MetricBuild metricBuilder.name.foreach(segment => jsonGenerator.writeString(convertNullToString(segment))) } jsonGenerator.writeEndArray() - val dataType = metricBuilder.metricType match { - case UnlatchedCounter => "unlatched_counter" - case CounterType => "counter" - case CounterishGaugeType => "counterish_gauge" - case GaugeType => "gauge" - case HistogramType => "histogram" - } + val dataType = metricBuilder.metricType.toJsonString jsonGenerator.writeStringField("kind", dataType) jsonGenerator.writeObjectFieldStart("source") jsonGenerator.writeStringField("class", metricBuilder.sourceClass.getOrElse("Unspecified")) From ada958f3f19e436f0c83157b4bec1b89298092de Mon Sep 17 00:00:00 2001 From: Lily He Date: Wed, 2 Mar 2022 23:53:28 +0000 Subject: [PATCH 337/409] finagle-core: Show LB panic mode in Stack Registry and loadbalancers.json Problem: TwitterServer doesn't show which panic mode the load balancer is using. Solution: Register the load balancer panic mode in TwitterServer admin page. The panic mode can be viewed in the client stack registry and in /admin/balancers.json. JIRA Issues: CSL-11618 Differential Revision: https://phabricator.twitter.biz/D831500 --- .../com/twitter/server/view/BalancerHtmlView.scala | 1 + .../twitter/server/view/BalancersJsonView.scala | 2 ++ .../server/view/BalancersJsonViewTest.scala | 14 ++++++++++---- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/view/BalancerHtmlView.scala b/server/src/main/scala/com/twitter/server/view/BalancerHtmlView.scala index 18640238..a5ac474f 100644 --- a/server/src/main/scala/com/twitter/server/view/BalancerHtmlView.scala +++ b/server/src/main/scala/com/twitter/server/view/BalancerHtmlView.scala @@ -20,6 +20,7 @@ private[server] class BalancerHtmlView(balancer: Option[Metadata], routePath: St | Total pending requests${md.totalPending} | Total load${md.totalLoad} | Size${md.size} + | Panic Mode${md.panicMode} | More details | """.stripMargin diff --git a/server/src/main/scala/com/twitter/server/view/BalancersJsonView.scala b/server/src/main/scala/com/twitter/server/view/BalancersJsonView.scala index f3bf4ca9..d58fb107 100644 --- a/server/src/main/scala/com/twitter/server/view/BalancersJsonView.scala +++ b/server/src/main/scala/com/twitter/server/view/BalancersJsonView.scala @@ -15,6 +15,7 @@ private object BalancersJsonView { totalPending: Int, totalLoad: Double, size: Int, + panicMode: String, additionalInfo: Map[String, Any]) def convertMetadata(metadata: Metadata): Balancer = @@ -29,6 +30,7 @@ private object BalancersJsonView { totalPending = metadata.totalPending, totalLoad = metadata.totalLoad, size = metadata.size, + panicMode = metadata.panicMode, additionalInfo = metadata.additionalInfo ) ) diff --git a/server/src/test/scala/com/twitter/server/view/BalancersJsonViewTest.scala b/server/src/test/scala/com/twitter/server/view/BalancersJsonViewTest.scala index 0a57f98b..8fbf077b 100644 --- a/server/src/test/scala/com/twitter/server/view/BalancersJsonViewTest.scala +++ b/server/src/test/scala/com/twitter/server/view/BalancersJsonViewTest.scala @@ -1,10 +1,15 @@ package com.twitter.server.view -import com.twitter.finagle.{Http, Name, Service} -import com.twitter.finagle.http.{Request, Response} -import com.twitter.finagle.loadbalancer.{BalancerRegistry, Balancers} +import com.twitter.finagle.Http +import com.twitter.finagle.Name +import com.twitter.finagle.Service +import com.twitter.finagle.http.Request +import com.twitter.finagle.http.Response +import com.twitter.finagle.loadbalancer.BalancerRegistry +import com.twitter.finagle.loadbalancer.Balancers import com.twitter.server.util.JsonUtils -import com.twitter.util.{Await, Future} +import com.twitter.util.Await +import com.twitter.util.Future import org.scalatest.funsuite.AnyFunSuite class BalancersJsonViewTest extends AnyFunSuite { @@ -35,6 +40,7 @@ class BalancersJsonViewTest extends AnyFunSuite { | "total_pending" : 0, | "total_load" : 0.0, | "size" : 1, + | "panic_mode" : "MajorityUnhealthy", | "additional_info" : { } | } | } From 3bb02e4327c407e1ecf9f6f136038d83809941ae Mon Sep 17 00:00:00 2001 From: Helen Woldesenbet Date: Fri, 11 Mar 2022 19:54:35 +0000 Subject: [PATCH 338/409] 3rdparty: jackson upgrade to 2.13.2 Problem We are on jackson 2.13.1 but we want to upgrade to a new version. Solution Upgrade to 2.13.2 JIRA Issues: CSL-11797 Differential Revision: https://phabricator.twitter.biz/D848592 --- CHANGELOG.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 5db97add..8ce472d0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,12 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Runtime Behavior Changes +~~~~~~~~~~~~~~~~~~~~~~~~ + +* Bump version of Jackson to 2.13.2. ``PHAB_ID=D848592`` + + 22.2.0 ------ From b2ea17099b2d40239012c169dab7cc828453d4d7 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Mon, 21 Mar 2022 18:59:35 +0000 Subject: [PATCH 339/409] inject-server: Throw an UnsupportedOperationException when DeprecatedLogging#log is accessed Problem The `c.t.inject.server.DeprecatedLogging` trait was introduced to the Finatra inject TwitterServer trait in order to provide a backward-compatible shim layer for Finatra services as they may have been setting the util-logging Flags and complete removal of the util-logging Logging trait would have removed the Flag definitions which would cause servers to fail startup. The shim layer was provided such that the Flags would still be defined even though they do nothing. And the logger instance was redefined to be marked `@deprecated` as a signal to users that it should not be used. However, users still mistakenly use it since logging in a Finatra application can be confusing due to the prevalence of the JUL logging library, `util/util-logging` in the stack. The stack is slowly moving away from the JUL logging of `util/util-logging` and to the SLF4J-API but nonetheless it is too nuanced to be obvious to most users right away. Solution Make accessing the `log` instance throw an `UnsupportedOperationException`. This instance should not be used for logging and the `DeprecatedLogging` trait is slated for removal. Hopefully throwing the exception prevents new incorrect usages from being introduced such that we can continue to unwind the removal of the `DeprecatedLogging` trait. JIRA Issues: CSL-7374 Differential Revision: https://phabricator.twitter.biz/D854365 --- CHANGELOG.rst | 3 +++ .../src/main/scala/com/twitter/server/logging/Logging.scala | 3 +++ 2 files changed, 6 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8ce472d0..785968b2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +* Update the twitter-server/slf4j-jdk14 Logging trait to ensure it defines a Logger to + handle the cases where a class the trait is mixed into has redefined the logger. ``PHAB_ID=D854365`` + Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/slf4j-jdk14/src/main/scala/com/twitter/server/logging/Logging.scala b/slf4j-jdk14/src/main/scala/com/twitter/server/logging/Logging.scala index 90fbfb9f..52d813f7 100644 --- a/slf4j-jdk14/src/main/scala/com/twitter/server/logging/Logging.scala +++ b/slf4j-jdk14/src/main/scala/com/twitter/server/logging/Logging.scala @@ -6,6 +6,9 @@ import java.util.logging.Logger trait Logging extends ctl.Logging { self: App => + /** ensure when this trait is used there is a defined logger */ + override lazy val log: ctl.Logger = ctl.Logger(name) + /** * Note: we are applying the `defaultFormatter` to any * configured handlers on the ROOT logger in the constructor From 9cd4550915a43d0f8077475b812236dfd6eafec1 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Wed, 23 Mar 2022 17:10:03 +0000 Subject: [PATCH 340/409] finagle-stats-core: Reshape MetricsView to support dimensional metrics Problem We need to be able to export dimensional metrics but we don't have a way to emit more than a name and a number from our MetricsView. Solution Use our *Snapshot types in MetricsView so we include more info about the metric itself. This lets our exporters know more about the metric so they can properly emit them. JIRA Issues: CSL-11616 Differential Revision: https://phabricator.twitter.biz/D841988 --- .../scala/com/twitter/server/util/MetricSource.scala | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/util/MetricSource.scala b/server/src/main/scala/com/twitter/server/util/MetricSource.scala index 79e0b276..358f1bf7 100644 --- a/server/src/main/scala/com/twitter/server/util/MetricSource.scala +++ b/server/src/main/scala/com/twitter/server/util/MetricSource.scala @@ -1,10 +1,12 @@ package com.twitter.server.util import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.stats.{StatEntry, StatsRegistry} +import com.twitter.finagle.stats.StatEntry +import com.twitter.finagle.stats.StatsRegistry import com.twitter.finagle.util.LoadService import com.twitter.server.util.MetricSource.MetricTypeInfo -import com.twitter.util.{Duration, Time} +import com.twitter.util.Duration +import com.twitter.util.Time import java.util.concurrent.atomic.AtomicReference private[server] object MetricSource { @@ -30,7 +32,9 @@ private[server] class MetricSource( val last = lastRefresh.get() if (last != null && Time.now - last > refreshInterval && lastRefresh.compareAndSet(last, null)) { - val newStats = registry().foldLeft(Map[String, StatEntry]()) { (map, r) => map ++ r() } + val newStats = registry().foldLeft(Map[String, StatEntry]()) { (map, r) => + map ++ r().iterator + } underlying = newStats lastRefresh.set(Time.now) } From 3554dff68eeabeb61991846fdc367f987d61b067 Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Fri, 25 Mar 2022 19:25:12 +0000 Subject: [PATCH 341/409] util-stats: more support for creating a Stat expression **Problem/Solution** Move `HistogramComponent` to its own class **Problem/Solution** Provide default percentiles in `HistogramComponent` **Problem** Percentile is not a component of `HistogramComponent`, to create a Stat expression, users need to pass either a `HistogramComponent` or a percentile as `Double`. **Solution** Introduce `Percentile` as a new `HistogramComponent` and refactor the API to create a Stat expression to take a `HistogramComponent`. This makes it easier for us to introduce a new API to create a list of Stat expressions with different `HistogramComponent` (will be done in a separate phab). JIRA Issues: CSL-11828 Differential Revision: https://phabricator.twitter.biz/D855103 --- .../server/handler/exp/MetricExpressionHandlerTest.scala | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala index 853e1992..0e00abc0 100644 --- a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala @@ -9,6 +9,7 @@ import com.twitter.finagle.stats.exp.Expression import com.twitter.finagle.stats.exp.ExpressionSchema import com.twitter.finagle.stats.exp.ExpressionSchemaKey import com.twitter.finagle.stats.exp.GreaterThan +import com.twitter.finagle.stats.exp.HistogramComponent import com.twitter.finagle.stats.exp.MonotoneThresholds import com.twitter.finagle.stats.InMemoryStatsReceiver import com.twitter.finagle.stats.MetricBuilder @@ -47,7 +48,8 @@ class MetricExpressionHandlerTest extends AnyFunSuite { .withNamespace("path", "to", "tenantName") val latencyP99 = - ExpressionSchema("latency_p99", Expression(latencyMb, Right(0.99))).withNamespace("tenantName") + ExpressionSchema("latency_p99", Expression(latencyMb, HistogramComponent.Percentile(0.99))) + .withNamespace("tenantName") val failureExpression = ExpressionSchema("failures", Expression(failuresMb, true)) @@ -276,7 +278,7 @@ class MetricExpressionHandlerTest extends AnyFunSuite { test("translate histogram expressions - components") { - val latencyMin = ExpressionSchema("min", Expression(latencyMb, Left(Expression.Min))) + val latencyMin = ExpressionSchema("min", Expression(latencyMb, HistogramComponent.Min)) val min = MetricExpressionHandler.translateToQuery( latencyMin.expr, shouldRate = false, From 849040714cde7e99c169137230e1df4639f7362e Mon Sep 17 00:00:00 2001 From: Joy Bestourous Date: Tue, 29 Mar 2022 16:59:03 +0000 Subject: [PATCH 342/409] {finagle, scrooge, twitter-server}: add jbestourous to PROJECT files Problem/Solution: add jbestourous to some PROJECT files Differential Revision: https://phabricator.twitter.biz/D863139 --- PROJECT.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/PROJECT.yml b/PROJECT.yml index 81c55540..32a6436f 100644 --- a/PROJECT.yml +++ b/PROJECT.yml @@ -4,6 +4,7 @@ owners: - banderson - hallam - ibennett + - jbestourous - mnakamura - roanta - yufang From 603693fbfeee7ab9aae3203f53522fbafc998f5e Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Tue, 29 Mar 2022 18:34:34 +0000 Subject: [PATCH 343/409] Twitter-oss: Prepare OSS libraries for release 22.3.0 Problem We want to release the next versions of our Twitter OSS libraries 22.3.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D863328 --- CHANGELOG.rst | 3 +++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 785968b2..14fb1031 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +22.3.0 +------ + * Update the twitter-server/slf4j-jdk14 Logging trait to ensure it defines a Logger to handle the cases where a class the trait is mixed into has redefined the logger. ``PHAB_ID=D854365`` diff --git a/README.md b/README.md index c4c5dc37..320da6d9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index fe3117aa..8c123590 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "22.3.0-SNAPSHOT" +val releaseVersion = "22.3.0" val logbackVersion = "1.2.8" val opencensusVersion = "0.19.1" From 3ab0a5bd911a20ca83cb68a9197691851f7157ab Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Tue, 29 Mar 2022 19:23:38 +0000 Subject: [PATCH 344/409] Twitter-oss: update OSS libraries post-release to 22.4.0 Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 22.4.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D863501 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 320da6d9..c4c5dc37 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 8c123590..9d707b9d 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "22.3.0" +val releaseVersion = "22.4.0-SNAPSHOT" val logbackVersion = "1.2.8" val opencensusVersion = "0.19.1" From 79fc647407c094573d70df90823f68c957082b65 Mon Sep 17 00:00:00 2001 From: Lily He Date: Sat, 2 Apr 2022 00:32:12 +0000 Subject: [PATCH 345/409] twitter-server: Sort expressions in Metric Expression Handler Test Problem: The sbt build fails on MetricExpressionHandlerTest "Get all expressions" because the order of the elements in the expressions array varies. Solution: Sort the expressions by "name" alphabetically. JIRA Issues: CSL-11879 Differential Revision: https://phabricator.twitter.biz/D863784 --- .../server/util/MetricSchemaSource.scala | 19 +- .../exp/MetricExpressionHandlerTest.scala | 218 +++++++++--------- 2 files changed, 125 insertions(+), 112 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/util/MetricSchemaSource.scala b/server/src/main/scala/com/twitter/server/util/MetricSchemaSource.scala index 8fca431e..d7326fbd 100644 --- a/server/src/main/scala/com/twitter/server/util/MetricSchemaSource.scala +++ b/server/src/main/scala/com/twitter/server/util/MetricSchemaSource.scala @@ -1,8 +1,10 @@ package com.twitter.server.util import com.twitter.finagle.stats.exp.ExpressionSchema -import com.twitter.finagle.stats.{MetricBuilder, SchemaRegistry} +import com.twitter.finagle.stats.MetricBuilder +import com.twitter.finagle.stats.SchemaRegistry import com.twitter.finagle.util.LoadService +import scala.collection.immutable.ListMap private[server] object MetricSchemaSource { lazy val registry: Seq[SchemaRegistry] = LoadService[SchemaRegistry]() @@ -11,9 +13,12 @@ private[server] object MetricSchemaSource { /** * A map from stats names to [[com.twitter.finagle.stats.StatEntry StatsEntries]] * which allows for stale StatEntries up to `refreshInterval`. + * @param registry Seq of schema registries + * @param sort if true, sort ExpressionSchema by name for deterministic test outputs */ private[server] class MetricSchemaSource( - registry: Seq[SchemaRegistry] = MetricSchemaSource.registry) { + registry: Seq[SchemaRegistry] = MetricSchemaSource.registry, + sort: Boolean = false) { /** * Indicates whether or not the MetricSource is using latched Counters. @@ -48,6 +53,14 @@ private[server] class MetricSchemaSource( } def expressionList: Iterable[ExpressionSchema] = synchronized { - registry.foldLeft(IndexedSeq[ExpressionSchema]()) { (seq, r) => seq ++ r.expressions().values } + if (!sort) { + registry.foldLeft(IndexedSeq[ExpressionSchema]()) { (seq, r) => + seq ++ r.expressions().values + } + } else { + registry.foldLeft(IndexedSeq[ExpressionSchema]()) { (seq, r) => + seq ++ ListMap(r.expressions().toSeq.sortBy(_._1.name): _*).values + } + } } } diff --git a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala index 0e00abc0..66ac4a21 100644 --- a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala @@ -2,18 +2,18 @@ package com.twitter.server.handler.exp import com.twitter.conversions.DurationOps._ import com.twitter.finagle.http.Request +import com.twitter.finagle.stats.InMemoryStatsReceiver +import com.twitter.finagle.stats.MetricBuilder import com.twitter.finagle.stats.MetricBuilder.CounterType import com.twitter.finagle.stats.MetricBuilder.GaugeType import com.twitter.finagle.stats.MetricBuilder.HistogramType +import com.twitter.finagle.stats.SchemaRegistry import com.twitter.finagle.stats.exp.Expression import com.twitter.finagle.stats.exp.ExpressionSchema import com.twitter.finagle.stats.exp.ExpressionSchemaKey import com.twitter.finagle.stats.exp.GreaterThan import com.twitter.finagle.stats.exp.HistogramComponent import com.twitter.finagle.stats.exp.MonotoneThresholds -import com.twitter.finagle.stats.InMemoryStatsReceiver -import com.twitter.finagle.stats.MetricBuilder -import com.twitter.finagle.stats.SchemaRegistry import com.twitter.server.handler.MetricExpressionHandler import com.twitter.server.util.AdminJsonConverter import com.twitter.server.util.JsonUtils @@ -77,7 +77,7 @@ class MetricExpressionHandlerTest extends AnyFunSuite { _root_.com.twitter.finagle.stats.exp.ExpressionSchema ] = expressionSchemaMap } - val expressionSource = new MetricSchemaSource(Seq(expressionRegistry)) + val expressionSource = new MetricSchemaSource(Seq(expressionRegistry), sort = true) val expressionHandler = new MetricExpressionHandler(expressionSource) val latchedRegistry = new SchemaRegistry { @@ -115,111 +115,111 @@ class MetricExpressionHandlerTest extends AnyFunSuite { test("Get all expressions") { val expectedResponse = """ - |{ - | "@version": 1.1, - | "counters_latched": false, - | "expressions": [ - | { - | "bounds": { - | "kind": "unbounded" - | }, - | "description": "Unspecified", - | "expression": "latency.p99", - | "labels": { - | "bucket": "p99", - | "process_path": "Unspecified", - | "role": "NoRoleSpecified", - | "service_name": "Unspecified" - | }, - | "name": "latency_p99", - | "namespaces": [ - | "tenantName" - | ], - | "unit": "Unspecified" - | }, - | { - | "bounds": { - | "kind": "unbounded" - | }, - | "description": "Unspecified", - | "expression": "srv/pending", - | "labels": { - | "process_path": "Unspecified", - | "role": "NoRoleSpecified", - | "service_name": "Unspecified" - | }, - | "name": "pending", - | "unit": "Unspecified" - | }, - | { - | "bounds": { - | "kind": "unbounded" - | }, - | "description": "Unspecified", - | "expression": "plus(success,failures)", - | "labels": { - | "process_path": "Unspecified", - | "role": "NoRoleSpecified", - | "service_name": "Unspecified" - | }, - | "name": "throughput", - | "namespaces": [ - | "path", - | "to", - | "tenantName" - | ], - | "unit": "Unspecified" - | }, - | { - | "bounds": { - | "kind": "unbounded" - | }, - | "description": "Unspecified", - | "expression": "failures/*", - | "labels": { - | "process_path": "Unspecified", - | "role": "NoRoleSpecified", - | "service_name": "Unspecified" - | }, - | "name": "failures", - | "unit": "Unspecified" - | }, - | { - | "bounds": { - | "kind": "unbounded" - | }, - | "description": "Unspecified", - | "expression": "jvm/uptime", - | "labels": { - | "process_path": "Unspecified", - | "role": "NoRoleSpecified", - | "service_name": "Unspecified" - | }, - | "name": "uptime", - | "unit": "Unspecified" - | }, - | { - | "bounds": { - | "bad_threshold": 99.5, - | "good_threshold": 99.97, - | "kind": "monotone", - | "lower_bound_inclusive": null, - | "operator": ">", - | "upper_bound_exclusive": null - | }, - | "description": "Unspecified", - | "expression": "multiply(100.0,divide(success,plus(success,failures)))", - | "labels": { - | "process_path": "Unspecified", - | "role": "NoRoleSpecified", - | "service_name": "Unspecified" - | }, - | "name": "success_rate", - | "unit": "Unspecified" - | } - | ], - | "separator_char": "/" - |}""".stripMargin + |{ + | "@version": 1.1, + | "counters_latched": false, + | "expressions": [ + | { + | "bounds": { + | "kind": "unbounded" + | }, + | "description": "Unspecified", + | "expression": "failures/*", + | "labels": { + | "process_path": "Unspecified", + | "role": "NoRoleSpecified", + | "service_name": "Unspecified" + | }, + | "name": "failures", + | "unit": "Unspecified" + | }, + | { + | "bounds": { + | "kind": "unbounded" + | }, + | "description": "Unspecified", + | "expression": "latency.p99", + | "labels": { + | "bucket": "p99", + | "process_path": "Unspecified", + | "role": "NoRoleSpecified", + | "service_name": "Unspecified" + | }, + | "name": "latency_p99", + | "namespaces": [ + | "tenantName" + | ], + | "unit": "Unspecified" + | }, + | { + | "bounds": { + | "kind": "unbounded" + | }, + | "description": "Unspecified", + | "expression": "srv/pending", + | "labels": { + | "process_path": "Unspecified", + | "role": "NoRoleSpecified", + | "service_name": "Unspecified" + | }, + | "name": "pending", + | "unit": "Unspecified" + | }, + | { + | "bounds": { + | "bad_threshold": 99.5, + | "good_threshold": 99.97, + | "kind": "monotone", + | "lower_bound_inclusive": null, + | "operator": ">", + | "upper_bound_exclusive": null + | }, + | "description": "Unspecified", + | "expression": "multiply(100.0,divide(success,plus(success,failures)))", + | "labels": { + | "process_path": "Unspecified", + | "role": "NoRoleSpecified", + | "service_name": "Unspecified" + | }, + | "name": "success_rate", + | "unit": "Unspecified" + | }, + | { + | "bounds": { + | "kind": "unbounded" + | }, + | "description": "Unspecified", + | "expression": "plus(success,failures)", + | "labels": { + | "process_path": "Unspecified", + | "role": "NoRoleSpecified", + | "service_name": "Unspecified" + | }, + | "name": "throughput", + | "namespaces": [ + | "path", + | "to", + | "tenantName" + | ], + | "unit": "Unspecified" + | }, + | { + | "bounds": { + | "kind": "unbounded" + | }, + | "description": "Unspecified", + | "expression": "jvm/uptime", + | "labels": { + | "process_path": "Unspecified", + | "role": "NoRoleSpecified", + | "service_name": "Unspecified" + | }, + | "name": "uptime", + | "unit": "Unspecified" + | } + | ], + | "separator_char": "/" + |}""".stripMargin JsonUtils.assertJsonResponse( await(expressionHandler(testRequest)).contentString, From 7c107d864e5ca6b29c6b1f7f312675bd9d398a25 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Wed, 13 Apr 2022 01:47:58 +0000 Subject: [PATCH 346/409] twitter-server: Emit labels and the type of name in the metrics schema endpoint Problem We want to include label information and whether the metric in question is valid for dimensional export. Solution Export the info. JIRA Issues: CSL-11809 Differential Revision: https://phabricator.twitter.biz/D864694 --- .../handler/MetricMetadataQueryHandler.scala | 18 ++++--- .../server/util/MetricSchemaSerializer.scala | 50 +++++++++++++++--- .../MetricMetadataQueryHandlerTest.scala | 52 ++++++++++++++++--- .../util/MetricSchemaJsonModuleTest.scala | 15 +++--- 4 files changed, 108 insertions(+), 27 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/handler/MetricMetadataQueryHandler.scala b/server/src/main/scala/com/twitter/server/handler/MetricMetadataQueryHandler.scala index 23af1267..1e796165 100644 --- a/server/src/main/scala/com/twitter/server/handler/MetricMetadataQueryHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/MetricMetadataQueryHandler.scala @@ -1,11 +1,17 @@ package com.twitter.server.handler import com.twitter.finagle.Service -import com.twitter.finagle.http.{MediaType, Request, Response, Uri} -import com.twitter.finagle.stats.{StatsFormatter, metadataScopeSeparator} +import com.twitter.finagle.http.MediaType +import com.twitter.finagle.http.Request +import com.twitter.finagle.http.Response +import com.twitter.finagle.http.Uri +import com.twitter.finagle.stats.MetricBuilder +import com.twitter.finagle.stats.StatsFormatter +import com.twitter.finagle.stats.metadataScopeSeparator import com.twitter.io.Buf import com.twitter.server.util.HttpUtils.newResponse -import com.twitter.server.util.{AdminJsonConverter, MetricSchemaSource} +import com.twitter.server.util.AdminJsonConverter +import com.twitter.server.util.MetricSchemaSource import com.twitter.util.Future /** @@ -86,8 +92,8 @@ class MetricMetadataQueryHandler(source: MetricSchemaSource = new MetricSchemaSo private[this] val statsFormatter = StatsFormatter.default - private[this] def query(keys: Iterable[String]) = { - keys.flatMap(k => { + private[this] def query(keys: Iterable[String]): Iterable[MetricBuilder] = { + keys.flatMap { k => // histogram metric. if (k.lastIndexOf(statsFormatter.histogramSeparator) > k.lastIndexOf( metadataScopeSeparator())) { @@ -101,7 +107,7 @@ class MetricMetadataQueryHandler(source: MetricSchemaSource = new MetricSchemaSo } else { source.getSchema(k) } - }) + } } def apply(req: Request): Future[Response] = { diff --git a/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala b/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala index f40e889f..1dbf0d64 100644 --- a/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala +++ b/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala @@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.SerializerProvider import com.fasterxml.jackson.databind.ser.std.StdSerializer import com.twitter.finagle.stats.MetricBuilder.HistogramType import com.twitter.finagle.stats.MetricBuilder +import com.twitter.finagle.stats.MetricBuilder.Identity import com.twitter.finagle.stats.StatsFormatter import com.twitter.finagle.stats.metadataScopeSeparator @@ -12,6 +13,34 @@ object SchemaSerializer extends StdSerializer[MetricBuilder](classOf[MetricBuild private[this] val statsFormatter = StatsFormatter.default + private[this] def dimensionallyValid(identity: Identity): Boolean = identity match { + case Identity.Full(_, _) => true + case Identity.Hierarchical(_, _) => false + } + + private[this] def writeArray( + jsonGenerator: JsonGenerator, + name: String, + strings: Iterator[String] + ): Unit = { + jsonGenerator.writeArrayFieldStart(name) + strings.foreach(s => jsonGenerator.writeString(convertNullToString(s))) + jsonGenerator.writeEndArray() + } + + private[this] def writeDictionary( + jsonGenerator: JsonGenerator, + name: String, + entries: Iterable[(String, String)] + ): Unit = { + jsonGenerator.writeObjectFieldStart(name) + entries.foreach { + case (k, v) => + jsonGenerator.writeStringField(k, v) + } + jsonGenerator.writeEndObject() + } + /** * This custom serializer is used to convert MetricSchemas to JSON for the metric_metadata * endpoint. @@ -32,14 +61,19 @@ object SchemaSerializer extends StdSerializer[MetricBuilder](classOf[MetricBuild val formattedName = metricBuilder.name.mkString(metadataScopeSeparator()) jsonGenerator.writeStartObject() jsonGenerator.writeStringField("name", formattedName) - jsonGenerator.writeArrayFieldStart("relative_name") - if (metricBuilder.relativeName != Seq.empty) { - metricBuilder.relativeName.foreach(segment => - jsonGenerator.writeString(convertNullToString(segment))) - } else { - metricBuilder.name.foreach(segment => jsonGenerator.writeString(convertNullToString(segment))) - } - jsonGenerator.writeEndArray() + + writeArray( + jsonGenerator, + "relative_name", + (if (metricBuilder.relativeName.nonEmpty) metricBuilder.relativeName + else metricBuilder.name).iterator) + + writeDictionary(jsonGenerator, "labels", metricBuilder.identity.labels) + + jsonGenerator.writeBooleanField( + "dimensional_support", + dimensionallyValid(metricBuilder.identity)) + val dataType = metricBuilder.metricType.toJsonString jsonGenerator.writeStringField("kind", dataType) jsonGenerator.writeObjectFieldStart("source") diff --git a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala index 19cbca07..f08d5de5 100644 --- a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala @@ -1,11 +1,16 @@ package com.twitter.server.handler import com.twitter.finagle.http.Request -import com.twitter.finagle.stats.MetricBuilder.{CounterType, GaugeType, HistogramType} +import com.twitter.finagle.stats.MetricBuilder.CounterType +import com.twitter.finagle.stats.MetricBuilder.GaugeType +import com.twitter.finagle.stats.MetricBuilder.HistogramType import com.twitter.finagle.stats._ -import com.twitter.finagle.stats.exp.{ExpressionSchema, ExpressionSchemaKey} -import com.twitter.server.util.{JsonUtils, MetricSchemaSource} -import com.twitter.util.{Await, Duration} +import com.twitter.finagle.stats.exp.ExpressionSchema +import com.twitter.finagle.stats.exp.ExpressionSchemaKey +import com.twitter.server.util.JsonUtils +import com.twitter.server.util.MetricSchemaSource +import com.twitter.util.Await +import com.twitter.util.Duration import org.scalatest.funsuite.AnyFunSuite class MetricMetadataQueryHandlerTest extends AnyFunSuite { @@ -24,7 +29,7 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), metricType = CounterType, statsReceiver = null - ), + ).withLabels(Map("label1" -> "value1", "label2" -> "value2")).withDimensionalSupport, "your/fine/gauge" -> MetricBuilder( keyIndicator = false, @@ -45,7 +50,7 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), metricType = HistogramType, statsReceiver = null - ), + ).withLabels(labels = Map("label1" -> "value1")).withDimensionalSupport, "my/bad/null/counter" -> MetricBuilder( keyIndicator = true, @@ -111,6 +116,11 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { | { | "name" : "my/cool/counter", | "relative_name" : ["my","cool","counter"], + | "labels" : { + | "label1" : "value1", + | "label2" : "value2" + | }, + | "dimensional_support" : true, | "kind" : "counter", | "source" : { | "class": "finagle.stats.cool", @@ -125,6 +135,8 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { | { | "name" : "your/fine/gauge", | "relative_name" : ["your","fine","gauge"], + | "labels" : {}, + | "dimensional_support" : false, | "kind" : "gauge", | "source" : { | "class": "finagle.stats.your", @@ -139,6 +151,10 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { | { | "name" : "my/only/histo", | "relative_name" : ["my","only","histo"], + | "labels" : { + | "label1" : "value1" + | }, + | "dimensional_support" : true, | "kind" : "histogram", | "source" : { | "class": "Unspecified", @@ -166,6 +182,8 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { | { | "name" : "my/bad/null/counter", | "relative_name" : ["my","bad", "null", "counter"], + | "labels" : {}, + | "dimensional_support" : false, | "kind" : "counter", | "source" : { | "class": "finagle.stats.bad", @@ -187,6 +205,8 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { | { | "name" : "your/fine/gauge", | "relative_name" : ["your","fine","gauge"], + | "labels" : {}, + | "dimensional_support" : false, | "kind" : "gauge", | "source" : { | "class": "finagle.stats.your", @@ -207,6 +227,11 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { | { | "name" : "my/cool/counter", | "relative_name" : ["my","cool","counter"], + | "labels" : { + | "label1" : "value1", + | "label2" : "value2" + | }, + | "dimensional_support" : true, | "kind" : "counter", | "source" : { | "class": "finagle.stats.cool", @@ -221,6 +246,8 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { | { | "name" : "your/fine/gauge", | "relative_name" : ["your","fine","gauge"], + | "labels" : {}, + | "dimensional_support" : false, | "kind" : "gauge", | "source" : { | "class": "finagle.stats.your", @@ -242,6 +269,10 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { | { | "name" : "my/only/histo", | "relative_name" : ["my","only","histo"], + | "labels" : { + | "label1" : "value1" + | }, + | "dimensional_support" : true, | "kind" : "histogram", | "source" : { | "class": "Unspecified", @@ -275,6 +306,11 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { | { | "name" : "my/cool/counter", | "relative_name" : ["my","cool","counter"], + | "labels" : { + | "label1" : "value1", + | "label2" : "value2" + | }, + | "dimensional_support" : true, | "kind" : "counter", | "source" : { | "class": "finagle.stats.cool", @@ -289,6 +325,10 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { | { | "name" : "my/only/histo", | "relative_name" : ["my","only","histo"], + | "labels" : { + | "label1" : "value1" + | }, + | "dimensional_support" : true, | "kind" : "histogram", | "source" : { | "class": "Unspecified", diff --git a/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala b/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala index 5e0ba0d5..11910e13 100644 --- a/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala +++ b/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala @@ -1,13 +1,12 @@ package com.twitter.server.util import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.module.scala.{DefaultScalaModule, ScalaObjectMapper} -import com.twitter.finagle.stats.MetricBuilder.{ - CounterType, - CounterishGaugeType, - GaugeType, - HistogramType -} +import com.fasterxml.jackson.module.scala.DefaultScalaModule +import com.fasterxml.jackson.module.scala.ScalaObjectMapper +import com.twitter.finagle.stats.MetricBuilder.CounterType +import com.twitter.finagle.stats.MetricBuilder.CounterishGaugeType +import com.twitter.finagle.stats.MetricBuilder.GaugeType +import com.twitter.finagle.stats.MetricBuilder.HistogramType import com.twitter.finagle.stats._ import org.scalatest.funsuite.AnyFunSuite @@ -53,6 +52,8 @@ class MetricSchemaJsonModuleTest extends AnyFunSuite { Set( "name", "relative_name", + "labels", + "dimensional_support", "kind", "source", "description", From 088fe5a46238a5387788505ce65d8902b04c5880 Mon Sep 17 00:00:00 2001 From: Helen Woldesenbet Date: Wed, 13 Apr 2022 19:02:09 +0000 Subject: [PATCH 347/409] Twitter-oss: Prepare OSS libraries for release 22.4.0 Problem We want to release the next versions of our Twitter OSS libraries 22.4.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D874434 --- CHANGELOG.rst | 5 +++++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 14fb1031..44e762d0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +22.4.0 +------ + +No Changes + 22.3.0 ------ diff --git a/README.md b/README.md index c4c5dc37..320da6d9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 9d707b9d..72c27ef4 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "22.4.0-SNAPSHOT" +val releaseVersion = "22.4.0" val logbackVersion = "1.2.8" val opencensusVersion = "0.19.1" From 442ab49be2c6ee228556dbab7414328495f42821 Mon Sep 17 00:00:00 2001 From: Helen Woldesenbet Date: Mon, 18 Apr 2022 22:17:04 +0000 Subject: [PATCH 348/409] Twitter-oss: update OSS libraries post-release to 22.5.0 Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 22.5.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version Differential Revision: https://phabricator.twitter.biz/D876500 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 320da6d9..c4c5dc37 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 72c27ef4..b11323da 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "22.4.0" +val releaseVersion = "22.5.0-SNAPSHOT" val logbackVersion = "1.2.8" val opencensusVersion = "0.19.1" From 1244f361b99f138b03a9c3af0d9d61e06aa9b114 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Thu, 28 Apr 2022 19:06:20 +0000 Subject: [PATCH 349/409] twitter-server: Pass along the Allow headers in the TunableHandler Problem When we get a bad request we sometimes want to pass along which HTTP methods are valid, and we actually do say what that is but we fail to put it in the HTTP response. Solution Add it to the HTTP response. Differential Revision: https://phabricator.twitter.biz/D883951 --- .../server/handler/TunableHandler.scala | 17 +++++-- .../server/handler/TunableHandlerTest.scala | 46 +++++++++++++------ 2 files changed, 44 insertions(+), 19 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/handler/TunableHandler.scala b/server/src/main/scala/com/twitter/server/handler/TunableHandler.scala index 22d12baa..2bb66109 100644 --- a/server/src/main/scala/com/twitter/server/handler/TunableHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/TunableHandler.scala @@ -1,14 +1,22 @@ package com.twitter.server.handler import com.twitter.finagle.Service -import com.twitter.finagle.http.{Fields, MediaType, Method, Request, Response, Status} +import com.twitter.finagle.http.Fields +import com.twitter.finagle.http.MediaType +import com.twitter.finagle.http.Method +import com.twitter.finagle.http.Request +import com.twitter.finagle.http.Response +import com.twitter.finagle.http.Status import com.twitter.finagle.tunable.StandardTunableMap import com.twitter.io.Buf import com.twitter.server.util.AdminJsonConverter import com.twitter.server.util.HttpUtils._ import com.twitter.util.logging.Logger -import com.twitter.util.tunable.{JsonTunableMapper, TunableMap} -import com.twitter.util.{Future, Return, Throw} +import com.twitter.util.tunable.JsonTunableMapper +import com.twitter.util.tunable.TunableMap +import com.twitter.util.Future +import com.twitter.util.Return +import com.twitter.util.Throw import scala.collection.mutable /** @@ -86,6 +94,7 @@ class TunableHandler private[handler] (registeredIdsFn: () => Map[String, Tunabl newResponse( status = status, contentType = "text/plain;charset=UTF-8", + headers = headers, content = Buf.Utf8(content) ) @@ -164,7 +173,7 @@ class TunableHandler private[handler] (registeredIdsFn: () => Map[String, Tunabl case Throw(e) => respond(Status.BadRequest, s"Failed to parse JSON for PUT request: ${e.getMessage}") } - case unsupported => + case _ => respond(Status.BadRequest, s"Expected Content-Type ${MediaType.Json} for PUT request") } diff --git a/server/src/test/scala/com/twitter/server/handler/TunableHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/TunableHandlerTest.scala index 77cad562..ad1de633 100644 --- a/server/src/test/scala/com/twitter/server/handler/TunableHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/TunableHandlerTest.scala @@ -1,10 +1,16 @@ package com.twitter.server.handler import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.module.scala.{DefaultScalaModule, ScalaObjectMapper} +import com.fasterxml.jackson.module.scala.DefaultScalaModule +import com.fasterxml.jackson.module.scala.ScalaObjectMapper import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.http.{MediaType, Method, Request, Status} +import com.twitter.finagle.http.Fields.Allow +import com.twitter.finagle.http.MediaType +import com.twitter.finagle.http.Method +import com.twitter.finagle.http.Request +import com.twitter.finagle.http.Status import com.twitter.util.Await +import com.twitter.util.Awaitable import com.twitter.util.tunable.TunableMap import org.scalatest.funsuite.AnyFunSuite @@ -18,14 +24,24 @@ object TunableHandlerTest { class TunableHandlerTest extends AnyFunSuite { import TunableHandlerTest._ + def await[T](t: Awaitable[T]): T = Await.result(t, 5.seconds) + private[this] val mapper = new ObjectMapper with ScalaObjectMapper { registerModule(DefaultScalaModule) } - test("Non-{DELETE, GET, PUT} request returns 'MethodNotAllowed response") { + test("Non-{DELETE, GET, PUT} request returns 'MethodNotAllowed response and 'Allow' header") { + val handler = new TunableHandler + val resp = await(handler(Request(Method.Post, "/"))) + assert(resp.status == Status.MethodNotAllowed) + assert(resp.headerMap.get(Allow) == Some("GET, PUT, DELETE")) + } + + test("Non-{GET} request returns 'MethodNotAllowed response and 'Allow' header") { val handler = new TunableHandler - val resp = handler(Request(Method.Post, "/")) - assert(Await.result(resp, 1.second).status == Status.MethodNotAllowed) + val resp = await(handler(Request(Method.Post, TunableHandler.Path))) + assert(resp.status == Status.MethodNotAllowed) + assert(resp.headerMap.get(Allow) == Some("GET")) } def testJsonErrorHandling(method: Method): Unit = { @@ -34,7 +50,7 @@ class TunableHandlerTest extends AnyFunSuite { val req = Request(method, "/") req.contentType = MediaType.Csv val resp = handler(req) - assert(Await.result(resp, 1.second).contentString.contains("Expected Content-Type")) + assert(Await.result(resp).contentString.contains("Expected Content-Type")) } test(method + ": Returns response with error if JSON cannot be parsed") { @@ -43,7 +59,7 @@ class TunableHandlerTest extends AnyFunSuite { req.contentType = MediaType.Json req.contentString = "i am not valid json..." val resp = handler(req) - assert(Await.result(resp, 1.second).contentString.contains("Failed to parse JSON")) + assert(await(resp).contentString.contains("Failed to parse JSON")) } } @@ -53,7 +69,7 @@ class TunableHandlerTest extends AnyFunSuite { val req = Request(method, "/admin/tunables/foo") req.contentType = MediaType.Json req.contentString = """{"tunables": [] }""" - val resp = Await.result(handler(req), 1.second) + val resp = await(handler(req)) assert(resp.contentString.contains("TunableMap not found for id: foo")) assert(resp.status == Status.NotFound) } @@ -80,7 +96,7 @@ class TunableHandlerTest extends AnyFunSuite { val req = Request(Method.Get, "/admin/tunables/foo") - val resp = Await.result(handler(req), 1.second) + val resp = await(handler(req)) assert(resp.status == Status.Ok) val expected = """{ @@ -126,7 +142,7 @@ class TunableHandlerTest extends AnyFunSuite { val req = Request(Method.Get, "/admin/tunables") - val resp = Await.result(handler(req), 1.second) + val resp = await(handler(req)) assert(resp.status == Status.Ok) val expected = """[ @@ -182,7 +198,7 @@ class TunableHandlerTest extends AnyFunSuite { |}""".stripMargin val resp = handler(req) assert(map(TunableMap.Key[String]("test_id"))().contains("hello")) - assert(Await.result(resp, 1.second).status == Status.Ok) + assert(await(resp).status == Status.Ok) } test("PUT: Updates existing tunable with new value") { @@ -205,7 +221,7 @@ class TunableHandlerTest extends AnyFunSuite { assert(map(key)().contains("hello")) val resp = handler(req) assert(map(key)().contains("goodbye")) - assert(Await.result(resp, 1.second).status == Status.Ok) + assert(await(resp).status == Status.Ok) } test("PUT: Does not remove existing tunables") { @@ -231,7 +247,7 @@ class TunableHandlerTest extends AnyFunSuite { val resp = handler(req) assert(map(key1)().contains("goodbye")) assert(map(key2)().contains("i'd better stick around")) - assert(Await.result(resp, 1.second).status == Status.Ok) + assert(await(resp).status == Status.Ok) } test("DELETE: Removes specified existing tunables") { @@ -253,7 +269,7 @@ class TunableHandlerTest extends AnyFunSuite { |}""".stripMargin val resp = handler(req) assert(map(key)().isEmpty) - assert(Await.result(resp, 1.second).status == Status.Ok) + assert(await(resp).status == Status.Ok) } test("PUT: Does not remove existing tunables not present in update JSON") { @@ -279,6 +295,6 @@ class TunableHandlerTest extends AnyFunSuite { val resp = handler(req) assert(map(key1)().isEmpty) assert(map(key2)().contains("i'd better stick around")) - assert(Await.result(resp, 1.second).status == Status.Ok) + assert(await(resp).status == Status.Ok) } } From e4acf5fe144d59ebae92e465c8b80e717f96a2b3 Mon Sep 17 00:00:00 2001 From: Lily He Date: Tue, 3 May 2022 17:22:41 +0000 Subject: [PATCH 350/409] csl: Add fileOwner for CHANGELOG.rst Problem/Solution: Add the whole team as the file owner for the CHANGELOG.rst file for: finagle, finatra, scrooge, twitter-server, util Differential Revision: https://phabricator.twitter.biz/D886420 --- PROJECT.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/PROJECT.yml b/PROJECT.yml index 32a6436f..0580bb0d 100644 --- a/PROJECT.yml +++ b/PROJECT.yml @@ -10,3 +10,7 @@ owners: - yufang watchers: - csl-team:ldap + +fileOwners: + CHANGELOG.rst: + - csl-team:ldap From 142d4d26bf697f48366efa69d32973b0434fe5d6 Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Tue, 17 May 2022 04:07:58 +0000 Subject: [PATCH 351/409] twitter-server: omit undefined labels in expressions.json endpoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem/Solution Omit undefined labels Instead using “Unspecified”, or “NoRoleSpecified” in expressions.json endpoint. JIRA Issues: CSL-11652 Differential Revision: https://phabricator.twitter.biz/D887773 --- .../server/util/exp/ExpressionJson.scala | 26 ++-- .../exp/MetricExpressionHandlerTest.scala | 126 +++--------------- 2 files changed, 30 insertions(+), 122 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/util/exp/ExpressionJson.scala b/server/src/main/scala/com/twitter/server/util/exp/ExpressionJson.scala index bf5d3bda..03fef9c3 100644 --- a/server/src/main/scala/com/twitter/server/util/exp/ExpressionJson.scala +++ b/server/src/main/scala/com/twitter/server/util/exp/ExpressionJson.scala @@ -1,10 +1,12 @@ package com.twitter.server.util.exp -import com.fasterxml.jackson.core.{JsonGenerator, JsonParser} +import com.fasterxml.jackson.core.JsonGenerator +import com.fasterxml.jackson.core.JsonParser import com.fasterxml.jackson.databind.deser.std.StdDeserializer import com.fasterxml.jackson.databind.ser.std.StdSerializer -import com.fasterxml.jackson.databind.{DeserializationContext, SerializerProvider} -import com.twitter.finagle.stats.NoRoleSpecified +import com.fasterxml.jackson.databind.DeserializationContext +import com.fasterxml.jackson.databind.SerializerProvider +import com.twitter.finagle.stats.Unspecified import com.twitter.finagle.stats.exp._ /** @@ -28,14 +30,7 @@ object ExpressionJson { gen.writeObjectFieldStart("labels") - // we need this to evolve the expressions format compatibly - // in the future, we will only export labels that are set - val labels = Map( - ExpressionSchema.ProcessPath -> "Unspecified", - ExpressionSchema.ServiceName -> "Unspecified", - ExpressionSchema.Role -> NoRoleSpecified.toString - ) ++ expressionSchema.labels - for ((key, value) <- labels) { + for ((key, value) <- expressionSchema.labels) { gen.writeStringField(key, value) } gen.writeEndObject() @@ -48,10 +43,13 @@ object ExpressionJson { gen.writeStringField("expression", expressionSchema.exprQuery) - provider.defaultSerializeField("bounds", expressionSchema.bounds, gen) + if (expressionSchema.bounds != Unbounded.get) + provider.defaultSerializeField("bounds", expressionSchema.bounds, gen) - gen.writeStringField("description", expressionSchema.description) - gen.writeStringField("unit", expressionSchema.unit.toString) + if (expressionSchema.description != "Unspecified") + gen.writeStringField("description", expressionSchema.description) + if (expressionSchema.unit != Unspecified) + gen.writeStringField("unit", expressionSchema.unit.toString) gen.writeEndObject() } } diff --git a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala index 66ac4a21..c96b1a87 100644 --- a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala @@ -120,50 +120,24 @@ class MetricExpressionHandlerTest extends AnyFunSuite { | "counters_latched": false, | "expressions": [ | { - | "bounds": { - | "kind": "unbounded" - | }, - | "description": "Unspecified", | "expression": "failures/*", - | "labels": { - | "process_path": "Unspecified", - | "role": "NoRoleSpecified", - | "service_name": "Unspecified" - | }, - | "name": "failures", - | "unit": "Unspecified" + | "labels": {}, + | "name": "failures" | }, | { - | "bounds": { - | "kind": "unbounded" - | }, - | "description": "Unspecified", | "expression": "latency.p99", | "labels": { - | "bucket": "p99", - | "process_path": "Unspecified", - | "role": "NoRoleSpecified", - | "service_name": "Unspecified" + | "bucket": "p99" | }, | "name": "latency_p99", | "namespaces": [ | "tenantName" - | ], - | "unit": "Unspecified" + | ] | }, | { - | "bounds": { - | "kind": "unbounded" - | }, - | "description": "Unspecified", | "expression": "srv/pending", - | "labels": { - | "process_path": "Unspecified", - | "role": "NoRoleSpecified", - | "service_name": "Unspecified" - | }, - | "name": "pending", - | "unit": "Unspecified" + | "labels": {}, + | "name": "pending" | }, | { | "bounds": { @@ -174,48 +148,24 @@ class MetricExpressionHandlerTest extends AnyFunSuite { | "operator": ">", | "upper_bound_exclusive": null | }, - | "description": "Unspecified", | "expression": "multiply(100.0,divide(success,plus(success,failures)))", - | "labels": { - | "process_path": "Unspecified", - | "role": "NoRoleSpecified", - | "service_name": "Unspecified" - | }, - | "name": "success_rate", - | "unit": "Unspecified" + | "labels": {}, + | "name": "success_rate" | }, | { - | "bounds": { - | "kind": "unbounded" - | }, - | "description": "Unspecified", | "expression": "plus(success,failures)", - | "labels": { - | "process_path": "Unspecified", - | "role": "NoRoleSpecified", - | "service_name": "Unspecified" - | }, + | "labels": {}, | "name": "throughput", | "namespaces": [ | "path", | "to", | "tenantName" - | ], - | "unit": "Unspecified" + | ] | }, | { - | "bounds": { - | "kind": "unbounded" - | }, - | "description": "Unspecified", | "expression": "jvm/uptime", - | "labels": { - | "process_path": "Unspecified", - | "role": "NoRoleSpecified", - | "service_name": "Unspecified" - | }, - | "name": "uptime", - | "unit": "Unspecified" + | "labels": {}, + | "name": "uptime" | } | ], | "separator_char": "/" @@ -348,12 +298,8 @@ class MetricExpressionHandlerTest extends AnyFunSuite { | "expressions" : [ | { | "name" : "success_rate", - | "labels" : { - | "process_path" : "Unspecified", - | "service_name" : "Unspecified", - | "role" : "NoRoleSpecified" - | }, | "expression" : "multiply(100.0,divide(success,plus(success,failures)))", + | "labels": {}, | "bounds" : { | "kind" : "monotone", | "operator" : ">", @@ -361,9 +307,7 @@ class MetricExpressionHandlerTest extends AnyFunSuite { | "good_threshold" : 99.97, | "lower_bound_inclusive" : null, | "upper_bound_exclusive" : null - | }, - | "description" : "Unspecified", - | "unit" : "Unspecified" + | } | } | ] |}""".stripMargin @@ -384,33 +328,16 @@ class MetricExpressionHandlerTest extends AnyFunSuite { | { | "name" : "latency_p99", | "labels" : { - | "process_path" : "Unspecified", - | "service_name" : "Unspecified", - | "role" : "NoRoleSpecified", | "bucket": "p99" | }, | "namespaces" : ["tenantName"], - | "expression" : "latency.p99", - | "bounds" : { - | "kind" : "unbounded" - | }, - | "description" : "Unspecified", - | "unit" : "Unspecified" + | "expression" : "latency.p99" | }, | { | "name" : "throughput", - | "labels" : { - | "process_path" : "Unspecified", - | "service_name" : "Unspecified", - | "role" : "NoRoleSpecified" - | }, | "namespaces" : ["path","to","tenantName"], | "expression" : "plus(success,failures)", - | "bounds" : { - | "kind" : "unbounded" - | }, - | "description" : "Unspecified", - | "unit" : "Unspecified" + | "labels": {} | } | ] |}""".stripMargin @@ -431,18 +358,10 @@ class MetricExpressionHandlerTest extends AnyFunSuite { | { | "name" : "latency_p99", | "labels" : { - | "process_path" : "Unspecified", - | "service_name" : "Unspecified", - | "role" : "NoRoleSpecified", | "bucket": "p99" | }, | "namespaces" : ["tenantName"], - | "expression" : "latency.p99", - | "bounds" : { - | "kind" : "unbounded" - | }, - | "description" : "Unspecified", - | "unit" : "Unspecified" + | "expression" : "latency.p99" | } | ] |}""".stripMargin @@ -462,18 +381,9 @@ class MetricExpressionHandlerTest extends AnyFunSuite { | "expressions" : [ | { | "name" : "throughput", - | "labels" : { - | "process_path" : "Unspecified", - | "service_name" : "Unspecified", - | "role" : "NoRoleSpecified" - | }, | "namespaces" : ["path","to","tenantName"], | "expression" : "plus(success,failures)", - | "bounds" : { - | "kind" : "unbounded" - | }, - | "description" : "Unspecified", - | "unit" : "Unspecified" + | "labels": {} | } | ] |}""".stripMargin From 21c2378e2770c3a53faeacafd284b16178ab1abf Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Tue, 17 May 2022 22:09:40 +0000 Subject: [PATCH 352/409] util-stats: More reshaping of dimensional metrics support Problem Recent plans have made scopes more valuable for us and less things will likely be changed from scopes to labels. This is great news as that means most scopes are still valid. However, we need to account for that in our model which doesn't allow scopes for dimensional metrics. Solution Simply allow it. We change the identity to be the union of the hierarchical and dimensional names plus the labels and whether the metric is valid dimensionally. JIRA Issues: CSL-11965 Differential Revision: https://phabricator.twitter.biz/D891342 --- .../server/util/MetricSchemaSerializer.scala | 14 +++++------- .../MetricMetadataQueryHandlerTest.scala | 22 ++++++++++++++----- .../util/MetricSchemaJsonModuleTest.scala | 1 + 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala b/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala index 1dbf0d64..8b1c9e35 100644 --- a/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala +++ b/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala @@ -5,7 +5,6 @@ import com.fasterxml.jackson.databind.SerializerProvider import com.fasterxml.jackson.databind.ser.std.StdSerializer import com.twitter.finagle.stats.MetricBuilder.HistogramType import com.twitter.finagle.stats.MetricBuilder -import com.twitter.finagle.stats.MetricBuilder.Identity import com.twitter.finagle.stats.StatsFormatter import com.twitter.finagle.stats.metadataScopeSeparator @@ -13,11 +12,6 @@ object SchemaSerializer extends StdSerializer[MetricBuilder](classOf[MetricBuild private[this] val statsFormatter = StatsFormatter.default - private[this] def dimensionallyValid(identity: Identity): Boolean = identity match { - case Identity.Full(_, _) => true - case Identity.Hierarchical(_, _) => false - } - private[this] def writeArray( jsonGenerator: JsonGenerator, name: String, @@ -62,6 +56,10 @@ object SchemaSerializer extends StdSerializer[MetricBuilder](classOf[MetricBuild jsonGenerator.writeStartObject() jsonGenerator.writeStringField("name", formattedName) + jsonGenerator.writeStringField( + "dimensional_name", + metricBuilder.identity.dimensionalName.mkString(MetricBuilder.DimensionalNameScopeSeparator)) + writeArray( jsonGenerator, "relative_name", @@ -70,9 +68,7 @@ object SchemaSerializer extends StdSerializer[MetricBuilder](classOf[MetricBuild writeDictionary(jsonGenerator, "labels", metricBuilder.identity.labels) - jsonGenerator.writeBooleanField( - "dimensional_support", - dimensionallyValid(metricBuilder.identity)) + jsonGenerator.writeBooleanField("dimensional_support", !metricBuilder.identity.hierarchicalOnly) val dataType = metricBuilder.metricType.toJsonString jsonGenerator.writeStringField("kind", dataType) diff --git a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala index f08d5de5..18c555e7 100644 --- a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala @@ -29,7 +29,7 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), metricType = CounterType, statsReceiver = null - ).withLabels(Map("label1" -> "value1", "label2" -> "value2")).withDimensionalSupport, + ).withLabels(Map("label1" -> "value1", "label2" -> "value2")), "your/fine/gauge" -> MetricBuilder( keyIndicator = false, @@ -50,7 +50,7 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), metricType = HistogramType, statsReceiver = null - ).withLabels(labels = Map("label1" -> "value1")).withDimensionalSupport, + ).withLabels(labels = Map("label1" -> "value1")), "my/bad/null/counter" -> MetricBuilder( keyIndicator = true, @@ -115,6 +115,7 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { | "metrics" : [ | { | "name" : "my/cool/counter", + | "dimensional_name" : "my_cool_counter", | "relative_name" : ["my","cool","counter"], | "labels" : { | "label1" : "value1", @@ -134,9 +135,10 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { | }, | { | "name" : "your/fine/gauge", + | "dimensional_name" : "your_fine_gauge", | "relative_name" : ["your","fine","gauge"], | "labels" : {}, - | "dimensional_support" : false, + | "dimensional_support" : true, | "kind" : "gauge", | "source" : { | "class": "finagle.stats.your", @@ -150,6 +152,7 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { | }, | { | "name" : "my/only/histo", + | "dimensional_name" : "my_only_histo", | "relative_name" : ["my","only","histo"], | "labels" : { | "label1" : "value1" @@ -181,9 +184,10 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { | }, | { | "name" : "my/bad/null/counter", + | "dimensional_name" : "my_bad_null_counter", | "relative_name" : ["my","bad", "null", "counter"], | "labels" : {}, - | "dimensional_support" : false, + | "dimensional_support" : true, | "kind" : "counter", | "source" : { | "class": "finagle.stats.bad", @@ -204,9 +208,10 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { | "metrics" : [ | { | "name" : "your/fine/gauge", + | "dimensional_name" : "your_fine_gauge", | "relative_name" : ["your","fine","gauge"], | "labels" : {}, - | "dimensional_support" : false, + | "dimensional_support" : true, | "kind" : "gauge", | "source" : { | "class": "finagle.stats.your", @@ -226,6 +231,7 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { | "metrics" : [ | { | "name" : "my/cool/counter", + | "dimensional_name": "my_cool_counter", | "relative_name" : ["my","cool","counter"], | "labels" : { | "label1" : "value1", @@ -245,9 +251,10 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { | }, | { | "name" : "your/fine/gauge", + | "dimensional_name" : "your_fine_gauge", | "relative_name" : ["your","fine","gauge"], | "labels" : {}, - | "dimensional_support" : false, + | "dimensional_support" : true, | "kind" : "gauge", | "source" : { | "class": "finagle.stats.your", @@ -268,6 +275,7 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { | "metrics" : [ | { | "name" : "my/only/histo", + | "dimensional_name" : "my_only_histo", | "relative_name" : ["my","only","histo"], | "labels" : { | "label1" : "value1" @@ -305,6 +313,7 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { | "metrics" : [ | { | "name" : "my/cool/counter", + | "dimensional_name" : "my_cool_counter", | "relative_name" : ["my","cool","counter"], | "labels" : { | "label1" : "value1", @@ -324,6 +333,7 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { | }, | { | "name" : "my/only/histo", + | "dimensional_name" : "my_only_histo", | "relative_name" : ["my","only","histo"], | "labels" : { | "label1" : "value1" diff --git a/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala b/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala index 11910e13..3fc4b805 100644 --- a/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala +++ b/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala @@ -51,6 +51,7 @@ class MetricSchemaJsonModuleTest extends AnyFunSuite { private val topLevelFieldNameSet = Set( "name", + "dimensional_name", "relative_name", "labels", "dimensional_support", From 5207d5aa982cf4cbcef8043c41121e6c5f830029 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Tue, 24 May 2022 00:11:20 +0000 Subject: [PATCH 353/409] util: Remove the StatsReceiver field from MetricBuilder Problem MetricBuilder is a relatively nice representation of a metric definition with one exception: the statsReceiver field, which defines some behavior and isn't always possible to populate. This is primarily used for registering expressions but that is kind of fraught with danger: which `statsReceiver` do you use when registering an expression composed of multiple metrics? Solution Remove that field and instead use explicit registration of expressions with a StatsReceiver. This pattern is still not perfect but perhaps a step in the right direction and we can try to sort out how to get rid of this registration pain later. JIRA Issues: CSL-11900 Differential Revision: https://phabricator.twitter.biz/D871577 --- .../MetricMetadataQueryHandlerTest.scala | 18 ++++++------------ .../exp/MetricExpressionHandlerTest.scala | 8 ++++---- .../util/MetricSchemaJsonModuleTest.scala | 9 +++------ .../server/util/MetricSchemaSourceTest.scala | 16 ++++++++-------- 4 files changed, 21 insertions(+), 30 deletions(-) diff --git a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala index 18c555e7..1f7ba344 100644 --- a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala @@ -27,8 +27,7 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { name = Seq("my", "cool", "counter"), processPath = Some("dc/role/zone/service"), percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), - metricType = CounterType, - statsReceiver = null + metricType = CounterType ).withLabels(Map("label1" -> "value1", "label2" -> "value2")), "your/fine/gauge" -> MetricBuilder( @@ -41,15 +40,13 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { name = Seq("your", "fine", "gauge"), processPath = Some("dc/your_role/zone/your_service"), percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), - metricType = GaugeType, - statsReceiver = null + metricType = GaugeType ), "my/only/histo" -> MetricBuilder( name = Seq("my", "only", "histo"), percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), - metricType = HistogramType, - statsReceiver = null + metricType = HistogramType ).withLabels(labels = Map("label1" -> "value1")), "my/bad/null/counter" -> MetricBuilder( @@ -62,8 +59,7 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { name = Seq("my", "bad", null, "counter"), processPath = Some("dc/role/zone/service"), percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), - metricType = CounterType, - statsReceiver = null + metricType = CounterType ) ) @@ -460,8 +456,7 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { MetricBuilder( name = metricName.split("\\/"), percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), - metricType = CounterType, - statsReceiver = null + metricType = CounterType )) } else { Map( @@ -469,8 +464,7 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { MetricBuilder( name = metricName.split("\\/"), percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), - metricType = HistogramType, - statsReceiver = null + metricType = HistogramType )) } } diff --git a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala index c96b1a87..8faab058 100644 --- a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala @@ -30,11 +30,11 @@ class MetricExpressionHandlerTest extends AnyFunSuite { val sr = new InMemoryStatsReceiver - val successMb = MetricBuilder(name = Seq("success"), metricType = CounterType, statsReceiver = sr) + val successMb = MetricBuilder(name = Seq("success"), metricType = CounterType) val failuresMb = - MetricBuilder(name = Seq("failures"), metricType = CounterType, statsReceiver = sr) + MetricBuilder(name = Seq("failures"), metricType = CounterType) val latencyMb = - MetricBuilder(name = Seq("latency"), metricType = HistogramType, statsReceiver = sr) + MetricBuilder(name = Seq("latency"), metricType = HistogramType) val successRateExpression = ExpressionSchema( @@ -245,7 +245,7 @@ class MetricExpressionHandlerTest extends AnyFunSuite { test("translate expressions - gauges") { val connMb = - MetricBuilder(name = Seq("client", "connections"), metricType = GaugeType, statsReceiver = sr) + MetricBuilder(name = Seq("client", "connections"), metricType = GaugeType) val result = MetricExpressionHandler.translateToQuery( Expression(connMb), shouldRate = false, diff --git a/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala b/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala index 3fc4b805..a2a0a237 100644 --- a/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala +++ b/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala @@ -23,8 +23,7 @@ class MetricSchemaJsonModuleTest extends AnyFunSuite { name = Seq("my", "cool", "counter"), processPath = Some("dc/role/zone/service"), percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), - metricType = CounterType, - statsReceiver = null + metricType = CounterType ) private val gaugeSchema = MetricBuilder( @@ -37,15 +36,13 @@ class MetricSchemaJsonModuleTest extends AnyFunSuite { name = Seq("your", "fine", "gauge"), processPath = Some("dc/your_role/zone/your_service"), percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), - metricType = GaugeType, - statsReceiver = null + metricType = GaugeType ) private val histogramSchema = MetricBuilder( name = Seq("my", "only", "histo"), percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), - metricType = HistogramType, - statsReceiver = null + metricType = HistogramType ) private val topLevelFieldNameSet = diff --git a/server/src/test/scala/com/twitter/server/util/MetricSchemaSourceTest.scala b/server/src/test/scala/com/twitter/server/util/MetricSchemaSourceTest.scala index 7b34ea96..e5ce39cc 100644 --- a/server/src/test/scala/com/twitter/server/util/MetricSchemaSourceTest.scala +++ b/server/src/test/scala/com/twitter/server/util/MetricSchemaSourceTest.scala @@ -1,8 +1,11 @@ package com.twitter.server.util -import com.twitter.finagle.stats.MetricBuilder.{CounterType, GaugeType, HistogramType} +import com.twitter.finagle.stats.MetricBuilder.CounterType +import com.twitter.finagle.stats.MetricBuilder.GaugeType +import com.twitter.finagle.stats.MetricBuilder.HistogramType import com.twitter.finagle.stats._ -import com.twitter.finagle.stats.exp.{ExpressionSchema, ExpressionSchemaKey} +import com.twitter.finagle.stats.exp.ExpressionSchema +import com.twitter.finagle.stats.exp.ExpressionSchemaKey import org.scalatest.funsuite.AnyFunSuite class MetricSchemaSourceTest extends AnyFunSuite { @@ -19,8 +22,7 @@ class MetricSchemaSourceTest extends AnyFunSuite { name = Seq("my", "cool", "counter"), processPath = Some("dc/role/zone/service"), percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), - metricType = CounterType, - statsReceiver = null + metricType = CounterType ), "your/fine/gauge" -> MetricBuilder( @@ -33,15 +35,13 @@ class MetricSchemaSourceTest extends AnyFunSuite { name = Seq("your", "fine", "gauge"), processPath = Some("dc/your_role/zone/your_service"), percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), - metricType = GaugeType, - statsReceiver = null + metricType = GaugeType ), "my/only/histo" -> MetricBuilder( name = Seq("my", "only", "histo"), percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), - metricType = HistogramType, - statsReceiver = null + metricType = HistogramType ) ) From 962137faf1182db108946d940fbf762805162a4c Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Thu, 26 May 2022 20:29:44 +0000 Subject: [PATCH 354/409] [Bazel] Add empty BUILD file to prevent full build Problem ------- SQ query walks up to find the affected targets, and end up causing full build for various targets. Solution -------- This adds guard `BUILD.bazel` file for all toplevel directories that do not have a BUILD file. JIRA Issues: DPB-15983, DPB-14048 Differential Revision: https://phabricator.twitter.biz/D902602 --- BUILD.bazel | 1 + 1 file changed, 1 insertion(+) create mode 100644 BUILD.bazel diff --git a/BUILD.bazel b/BUILD.bazel new file mode 100644 index 00000000..dbd3d061 --- /dev/null +++ b/BUILD.bazel @@ -0,0 +1 @@ +# This prevents SQ query from grabbing //:all since it traverses up once to find a BUILD (DPB-14048) From de2a59404ea5ef6fce56b17b7f1786919b714152 Mon Sep 17 00:00:00 2001 From: Dorothy Ordogh Date: Thu, 2 Jun 2022 00:05:20 +0000 Subject: [PATCH 355/409] Revert "twitter-server: omit undefined labels in expressions.json endpoint" This reverts commit bb54e6d7ffa1031755b9dc0543bc6c8c6e0483db. JIRA Issues: CSL-12048, OBSERVE-35755 Differential Revision: https://phabricator.twitter.biz/D905849 --- .../server/util/exp/ExpressionJson.scala | 20 +-- .../exp/MetricExpressionHandlerTest.scala | 126 +++++++++++++++--- 2 files changed, 120 insertions(+), 26 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/util/exp/ExpressionJson.scala b/server/src/main/scala/com/twitter/server/util/exp/ExpressionJson.scala index 03fef9c3..4c6fcd2c 100644 --- a/server/src/main/scala/com/twitter/server/util/exp/ExpressionJson.scala +++ b/server/src/main/scala/com/twitter/server/util/exp/ExpressionJson.scala @@ -6,7 +6,7 @@ import com.fasterxml.jackson.databind.deser.std.StdDeserializer import com.fasterxml.jackson.databind.ser.std.StdSerializer import com.fasterxml.jackson.databind.DeserializationContext import com.fasterxml.jackson.databind.SerializerProvider -import com.twitter.finagle.stats.Unspecified +import com.twitter.finagle.stats.NoRoleSpecified import com.twitter.finagle.stats.exp._ /** @@ -30,7 +30,14 @@ object ExpressionJson { gen.writeObjectFieldStart("labels") - for ((key, value) <- expressionSchema.labels) { + // we need this to evolve the expressions format compatibly + // in the future, we will only export labels that are set + val labels = Map( + ExpressionSchema.ProcessPath -> "Unspecified", + ExpressionSchema.ServiceName -> "Unspecified", + ExpressionSchema.Role -> NoRoleSpecified.toString + ) ++ expressionSchema.labels + for ((key, value) <- labels) { gen.writeStringField(key, value) } gen.writeEndObject() @@ -43,13 +50,10 @@ object ExpressionJson { gen.writeStringField("expression", expressionSchema.exprQuery) - if (expressionSchema.bounds != Unbounded.get) - provider.defaultSerializeField("bounds", expressionSchema.bounds, gen) + provider.defaultSerializeField("bounds", expressionSchema.bounds, gen) - if (expressionSchema.description != "Unspecified") - gen.writeStringField("description", expressionSchema.description) - if (expressionSchema.unit != Unspecified) - gen.writeStringField("unit", expressionSchema.unit.toString) + gen.writeStringField("description", expressionSchema.description) + gen.writeStringField("unit", expressionSchema.unit.toString) gen.writeEndObject() } } diff --git a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala index 8faab058..d6ddd4b4 100644 --- a/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/exp/MetricExpressionHandlerTest.scala @@ -120,24 +120,50 @@ class MetricExpressionHandlerTest extends AnyFunSuite { | "counters_latched": false, | "expressions": [ | { + | "bounds": { + | "kind": "unbounded" + | }, + | "description": "Unspecified", | "expression": "failures/*", - | "labels": {}, - | "name": "failures" + | "labels": { + | "process_path": "Unspecified", + | "role": "NoRoleSpecified", + | "service_name": "Unspecified" + | }, + | "name": "failures", + | "unit": "Unspecified" | }, | { + | "bounds": { + | "kind": "unbounded" + | }, + | "description": "Unspecified", | "expression": "latency.p99", | "labels": { - | "bucket": "p99" + | "bucket": "p99", + | "process_path": "Unspecified", + | "role": "NoRoleSpecified", + | "service_name": "Unspecified" | }, | "name": "latency_p99", | "namespaces": [ | "tenantName" - | ] + | ], + | "unit": "Unspecified" | }, | { + | "bounds": { + | "kind": "unbounded" + | }, + | "description": "Unspecified", | "expression": "srv/pending", - | "labels": {}, - | "name": "pending" + | "labels": { + | "process_path": "Unspecified", + | "role": "NoRoleSpecified", + | "service_name": "Unspecified" + | }, + | "name": "pending", + | "unit": "Unspecified" | }, | { | "bounds": { @@ -148,24 +174,48 @@ class MetricExpressionHandlerTest extends AnyFunSuite { | "operator": ">", | "upper_bound_exclusive": null | }, + | "description": "Unspecified", | "expression": "multiply(100.0,divide(success,plus(success,failures)))", - | "labels": {}, - | "name": "success_rate" + | "labels": { + | "process_path": "Unspecified", + | "role": "NoRoleSpecified", + | "service_name": "Unspecified" + | }, + | "name": "success_rate", + | "unit": "Unspecified" | }, | { + | "bounds": { + | "kind": "unbounded" + | }, + | "description": "Unspecified", | "expression": "plus(success,failures)", - | "labels": {}, + | "labels": { + | "process_path": "Unspecified", + | "role": "NoRoleSpecified", + | "service_name": "Unspecified" + | }, | "name": "throughput", | "namespaces": [ | "path", | "to", | "tenantName" - | ] + | ], + | "unit": "Unspecified" | }, | { + | "bounds": { + | "kind": "unbounded" + | }, + | "description": "Unspecified", | "expression": "jvm/uptime", - | "labels": {}, - | "name": "uptime" + | "labels": { + | "process_path": "Unspecified", + | "role": "NoRoleSpecified", + | "service_name": "Unspecified" + | }, + | "name": "uptime", + | "unit": "Unspecified" | } | ], | "separator_char": "/" @@ -298,8 +348,12 @@ class MetricExpressionHandlerTest extends AnyFunSuite { | "expressions" : [ | { | "name" : "success_rate", + | "labels" : { + | "process_path" : "Unspecified", + | "service_name" : "Unspecified", + | "role" : "NoRoleSpecified" + | }, | "expression" : "multiply(100.0,divide(success,plus(success,failures)))", - | "labels": {}, | "bounds" : { | "kind" : "monotone", | "operator" : ">", @@ -307,7 +361,9 @@ class MetricExpressionHandlerTest extends AnyFunSuite { | "good_threshold" : 99.97, | "lower_bound_inclusive" : null, | "upper_bound_exclusive" : null - | } + | }, + | "description" : "Unspecified", + | "unit" : "Unspecified" | } | ] |}""".stripMargin @@ -328,16 +384,33 @@ class MetricExpressionHandlerTest extends AnyFunSuite { | { | "name" : "latency_p99", | "labels" : { + | "process_path" : "Unspecified", + | "service_name" : "Unspecified", + | "role" : "NoRoleSpecified", | "bucket": "p99" | }, | "namespaces" : ["tenantName"], - | "expression" : "latency.p99" + | "expression" : "latency.p99", + | "bounds" : { + | "kind" : "unbounded" + | }, + | "description" : "Unspecified", + | "unit" : "Unspecified" | }, | { | "name" : "throughput", + | "labels" : { + | "process_path" : "Unspecified", + | "service_name" : "Unspecified", + | "role" : "NoRoleSpecified" + | }, | "namespaces" : ["path","to","tenantName"], | "expression" : "plus(success,failures)", - | "labels": {} + | "bounds" : { + | "kind" : "unbounded" + | }, + | "description" : "Unspecified", + | "unit" : "Unspecified" | } | ] |}""".stripMargin @@ -358,10 +431,18 @@ class MetricExpressionHandlerTest extends AnyFunSuite { | { | "name" : "latency_p99", | "labels" : { + | "process_path" : "Unspecified", + | "service_name" : "Unspecified", + | "role" : "NoRoleSpecified", | "bucket": "p99" | }, | "namespaces" : ["tenantName"], - | "expression" : "latency.p99" + | "expression" : "latency.p99", + | "bounds" : { + | "kind" : "unbounded" + | }, + | "description" : "Unspecified", + | "unit" : "Unspecified" | } | ] |}""".stripMargin @@ -381,9 +462,18 @@ class MetricExpressionHandlerTest extends AnyFunSuite { | "expressions" : [ | { | "name" : "throughput", + | "labels" : { + | "process_path" : "Unspecified", + | "service_name" : "Unspecified", + | "role" : "NoRoleSpecified" + | }, | "namespaces" : ["path","to","tenantName"], | "expression" : "plus(success,failures)", - | "labels": {} + | "bounds" : { + | "kind" : "unbounded" + | }, + | "description" : "Unspecified", + | "unit" : "Unspecified" | } | ] |}""".stripMargin From 96aef3fe6e81cd8fd936ea43d8f7ca5550d6a8f8 Mon Sep 17 00:00:00 2001 From: Kyle Bahr Date: Thu, 2 Jun 2022 03:24:20 +0000 Subject: [PATCH 356/409] twitter-server/server: add global configuration option for AdminHttpServer Problem/Solution: The AdminHttpServer needs a hook for adding global settings such as security configuration JIRA Issues: CSL-11988 Differential Revision: https://phabricator.twitter.biz/D899304 --- .../com/twitter/server/AdminHttpServer.scala | 79 ++++++++++--- .../twitter/server/AdminHttpServerTest.scala | 110 ++++++++++++++++++ 2 files changed, 171 insertions(+), 18 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/AdminHttpServer.scala b/server/src/main/scala/com/twitter/server/AdminHttpServer.scala index 92cda850..fd015af7 100644 --- a/server/src/main/scala/com/twitter/server/AdminHttpServer.scala +++ b/server/src/main/scala/com/twitter/server/AdminHttpServer.scala @@ -1,30 +1,59 @@ package com.twitter.server -import com.twitter.app.{App, Flag} +import com.twitter.app.App +import com.twitter.app.Flag import com.twitter.finagle.client.ClientRegistry import com.twitter.finagle.filter.ServerAdmissionControl -import com.twitter.finagle.http.{HttpMuxer, Method, Request, Response, Route => HttpRoute} +import com.twitter.finagle.http.HttpMuxer +import com.twitter.finagle.http.Method +import com.twitter.finagle.http.Request +import com.twitter.finagle.http.Response +import com.twitter.finagle.http.{Route => HttpRoute} import com.twitter.finagle.server.ServerRegistry import com.twitter.finagle.stats.NullStatsReceiver import com.twitter.finagle.tracing.NullTracer import com.twitter.finagle.util.LoadService -import com.twitter.finagle.{Http, ListeningServer, NullServer, Service} +import com.twitter.finagle.Http +import com.twitter.finagle.ListeningServer +import com.twitter.finagle.NullServer +import com.twitter.finagle.Service import com.twitter.server.Admin.Grouping import com.twitter.server.filters.AdminThreadPoolFilter -import com.twitter.server.handler.{AdminHttpMuxHandler, LoggingHandler, NoLoggingHandler} +import com.twitter.server.handler.AdminHttpMuxHandler +import com.twitter.server.handler.LoggingHandler +import com.twitter.server.handler.NoLoggingHandler import com.twitter.server.lint.LoggingRules import com.twitter.server.util.HttpUtils -import com.twitter.server.view.{IndexView, NotFoundView} +import com.twitter.server.view.IndexView +import com.twitter.server.view.NotFoundView import com.twitter.util.lint.GlobalRules import com.twitter.util.registry.Library -import com.twitter.util.{Future, Monitor, Time} -import java.net.{InetSocketAddress, URLEncoder} +import com.twitter.util.Future +import com.twitter.util.Monitor +import com.twitter.util.Time +import java.net.InetSocketAddress +import java.net.URLEncoder import java.nio.charset.StandardCharsets import org.slf4j.LoggerFactory import scala.language.reflectiveCalls object AdminHttpServer { + type GlobalConfigurator = Http.Server => Http.Server + + @volatile private var globalConfigurator: Option[GlobalConfigurator] = None + + private[twitter] def setGlobalConfigurator(config: GlobalConfigurator): Unit = { + globalConfigurator = Some(config) + } + + // Only known use case is for testing + private[twitter] def resetGlobalConfigurator(): Unit = { + globalConfigurator = None + } + + private[twitter] def getGlobalConfigurator: Option[GlobalConfigurator] = globalConfigurator + /** * The name used for the finagle server. */ @@ -221,6 +250,19 @@ trait AdminHttpServer { self: App with Stats => */ protected def configureAdminHttpServer(server: Http.Server): Http.Server = server + /** + * This method allows applying a global configuration, such as adding + * security/server authentication + * @param server the locally configured server + * @return the server with applied global settings + */ + private def runGlobalConfigurator(server: Http.Server): Http.Server = { + AdminHttpServer.getGlobalConfigurator match { + case Some(f) => f(server) + case None => server + } + } + private[this] def updateMuxer(): Unit = { // create a service which multiplexes across all endpoints. val localMuxer = allRoutes.foldLeft(new HttpMuxer) { @@ -293,17 +335,18 @@ trait AdminHttpServer { self: App with Stats => } log.info(s"Serving admin http on ${adminPort()}") - adminHttpServer = configureAdminHttpServer( - Http.server - .configured(Http.Netty4Impl) - .withStatsReceiver(NullStatsReceiver) - .withTracer(NullTracer) - .withMonitor(loggingMonitor) - .withLabel(ServerName) - // disable admission control, since we want the server to report stats - // especially when it's in a bad state. - .configured(ServerAdmissionControl.Param(false)) - ).serve(adminPort(), new NotFoundView andThen adminHttpMuxer) + adminHttpServer = runGlobalConfigurator( + configureAdminHttpServer( + Http.server + .configured(Http.Netty4Impl) + .withStatsReceiver(NullStatsReceiver) + .withTracer(NullTracer) + .withMonitor(loggingMonitor) + .withLabel(ServerName) + // disable admission control, since we want the server to report stats + // especially when it's in a bad state. + .configured(ServerAdmissionControl.Param(false)) + )).serve(adminPort(), new NotFoundView andThen adminHttpMuxer) closeOnExitLast(adminHttpServer) Library.register(libraryName, Map.empty) diff --git a/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala b/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala index 0cc808df..0037b7bc 100644 --- a/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala +++ b/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala @@ -234,4 +234,114 @@ class AdminHttpServerTest extends AnyFunSuite with Eventually with IntegrationPa } } + test("Default global configurator is empty") { + + var globalConfiguredServer: Option[Http.Server] = None + + class SpyServer extends TestTwitterServer { + var localConfiguredServer: Option[Http.Server] = None + + override def configureAdminHttpServer(server: Http.Server): Http.Server = { + localConfiguredServer = Some( + super + .configureAdminHttpServer(server)) + localConfiguredServer.get + } + } + + val server = new SpyServer { + override def main(): Unit = { + // not used + } + } + server.main(args = Array.empty[String]) + + assert(globalConfiguredServer == None) + assert(server.localConfiguredServer != None) + } + + test("Set global configurator gets run") { + + var globalConfiguredServer: Option[Http.Server] = None + + class SpyServer extends TestTwitterServer { + var localConfiguredServer: Option[Http.Server] = None + + override def configureAdminHttpServer(server: Http.Server): Http.Server = { + localConfiguredServer = Some( + super + .configureAdminHttpServer(server)) + localConfiguredServer.get + } + } + + AdminHttpServer.setGlobalConfigurator { srv => + globalConfiguredServer = Some(srv) + srv + } + + val server = new SpyServer { + override def main(): Unit = { + // not used + } + } + server.main(args = Array.empty[String]) + + assert(globalConfiguredServer == server.localConfiguredServer) + assert(server.localConfiguredServer != None) + } + + test("Global configurator throws and does not recover") { + + var globalConfiguredServer: Option[Http.Server] = None + + class SpyServer extends TestTwitterServer { + + private[this] var expectedThrowable: Option[Throwable] = None + + def setExpectedThrowable(t: Throwable) = { + expectedThrowable = Some(t) + } + + var localConfiguredServer: Option[Http.Server] = None + + override def configureAdminHttpServer(server: Http.Server): Http.Server = { + localConfiguredServer = Some( + super + .configureAdminHttpServer(server)) + localConfiguredServer.get + } + + override protected def exitOnError(throwable: Throwable): Unit = { + expectedThrowable match { + case Some(t) => assert(throwable == t) + case None => super.exitOnError(throwable) + } + } + } + + val badException = new Exception("Bad") + AdminHttpServer.setGlobalConfigurator { _ => + throw badException + } + + val server = new SpyServer { + override def main(): Unit = { + // not used + } + } + + server.setExpectedThrowable(badException) + try { + server.main(args = Array.empty[String]) + // server will have attempted to exit with error and asserted that error == expected + // verify state is as expected + assert(globalConfiguredServer == None) + assert(server.localConfiguredServer != None) + } finally { + // Reset here because this is global and can cause other tests to fail + AdminHttpServer.resetGlobalConfigurator() + } + } + } From ec37229d4261de772008fd929b339703a5eab818 Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Thu, 2 Jun 2022 20:44:43 +0000 Subject: [PATCH 357/409] Add myself to owners For my past and future contributions, add myself as owners to some projects. Differential Revision: https://phabricator.twitter.biz/D906693 --- PROJECT.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/PROJECT.yml b/PROJECT.yml index 0580bb0d..00453376 100644 --- a/PROJECT.yml +++ b/PROJECT.yml @@ -5,6 +5,7 @@ owners: - hallam - ibennett - jbestourous + - jyan - mnakamura - roanta - yufang From 0a6b09343cd595954b5766fc2398232d40b8ae73 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Thu, 2 Jun 2022 23:29:50 +0000 Subject: [PATCH 358/409] [logging] Move twitter-server/slf4j-jdk14/src/main/scala to binary targets Problem The `twitter-server/slf4j-jdk14/src/main/scala` dependency is a runtime dependency which should only be specified on binary targets. Solution Move the dependency from library targets to the necessary runtime targets. JIRA Issues: CSL-10903 Differential Revision: https://phabricator.twitter.biz/D876651 --- slf4j-jdk14/src/main/scala/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/slf4j-jdk14/src/main/scala/BUILD b/slf4j-jdk14/src/main/scala/BUILD index 3f10e49a..2997bd1d 100644 --- a/slf4j-jdk14/src/main/scala/BUILD +++ b/slf4j-jdk14/src/main/scala/BUILD @@ -10,6 +10,7 @@ scala_library( strict_deps = True, tags = [ "bazel-compatible", + "logging_impl", "logging_impl_check_whitelisted_target", ], dependencies = [ From cc0207c7b4a4fe2c1c59cddfab79d1e76466f76a Mon Sep 17 00:00:00 2001 From: Kyle Bahr Date: Thu, 9 Jun 2022 06:05:03 +0000 Subject: [PATCH 359/409] Add a point to the AdminHttpServer to latch additional config hooks onto twitter-server: Add marker Param to the Stack for AdminHttpServer Problem/Solution: The AdminHttpServer is not differentiated from any other service that it administrates and is difficult to configure separately, should a user want to. This change adds a Stack.Param that marks a Stack as being a Admin interface should a user want to write additional configuration hooks that target this Stack.Param. JIRA Issues: CSL-11988 Differential Revision: https://phabricator.twitter.biz/D908894 --- .../com/twitter/server/AdminHttpServer.scala | 71 ++++++------- .../twitter/server/AdminHttpServerTest.scala | 100 +----------------- 2 files changed, 36 insertions(+), 135 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/AdminHttpServer.scala b/server/src/main/scala/com/twitter/server/AdminHttpServer.scala index fd015af7..533fc5a0 100644 --- a/server/src/main/scala/com/twitter/server/AdminHttpServer.scala +++ b/server/src/main/scala/com/twitter/server/AdminHttpServer.scala @@ -17,6 +17,7 @@ import com.twitter.finagle.Http import com.twitter.finagle.ListeningServer import com.twitter.finagle.NullServer import com.twitter.finagle.Service +import com.twitter.finagle.Stack import com.twitter.server.Admin.Grouping import com.twitter.server.filters.AdminThreadPoolFilter import com.twitter.server.handler.AdminHttpMuxHandler @@ -39,21 +40,6 @@ import scala.language.reflectiveCalls object AdminHttpServer { - type GlobalConfigurator = Http.Server => Http.Server - - @volatile private var globalConfigurator: Option[GlobalConfigurator] = None - - private[twitter] def setGlobalConfigurator(config: GlobalConfigurator): Unit = { - globalConfigurator = Some(config) - } - - // Only known use case is for testing - private[twitter] def resetGlobalConfigurator(): Unit = { - globalConfigurator = None - } - - private[twitter] def getGlobalConfigurator: Option[GlobalConfigurator] = globalConfigurator - /** * The name used for the finagle server. */ @@ -147,6 +133,24 @@ object AdminHttpServer { ) } +/** + * A marker stack param that this is an admin interface. This could be + * used by a StackTransformer or ServerParamsInjector to do specialized + * configuration of admin servers + */ +private[twitter] object AdminServerInterface { + + case class Param(isAdmin: Boolean) {} + + object Param { + implicit val param = Stack.Param(False) + } + + def True = Param(true) + def False = Param(false) + +} + trait AdminHttpServer { self: App with Stats => import AdminHttpServer._ @@ -250,19 +254,6 @@ trait AdminHttpServer { self: App with Stats => */ protected def configureAdminHttpServer(server: Http.Server): Http.Server = server - /** - * This method allows applying a global configuration, such as adding - * security/server authentication - * @param server the locally configured server - * @return the server with applied global settings - */ - private def runGlobalConfigurator(server: Http.Server): Http.Server = { - AdminHttpServer.getGlobalConfigurator match { - case Some(f) => f(server) - case None => server - } - } - private[this] def updateMuxer(): Unit = { // create a service which multiplexes across all endpoints. val localMuxer = allRoutes.foldLeft(new HttpMuxer) { @@ -335,18 +326,18 @@ trait AdminHttpServer { self: App with Stats => } log.info(s"Serving admin http on ${adminPort()}") - adminHttpServer = runGlobalConfigurator( - configureAdminHttpServer( - Http.server - .configured(Http.Netty4Impl) - .withStatsReceiver(NullStatsReceiver) - .withTracer(NullTracer) - .withMonitor(loggingMonitor) - .withLabel(ServerName) - // disable admission control, since we want the server to report stats - // especially when it's in a bad state. - .configured(ServerAdmissionControl.Param(false)) - )).serve(adminPort(), new NotFoundView andThen adminHttpMuxer) + adminHttpServer = configureAdminHttpServer( + Http.server + .configured(Http.Netty4Impl) + .withStatsReceiver(NullStatsReceiver) + .withTracer(NullTracer) + .withMonitor(loggingMonitor) + .withLabel(ServerName) + // disable admission control, since we want the server to report stats + // especially when it's in a bad state. + .configured(ServerAdmissionControl.Param(false)) + .configured(AdminServerInterface.True) + ).serve(adminPort(), new NotFoundView andThen adminHttpMuxer) closeOnExitLast(adminHttpServer) Library.register(libraryName, Map.empty) diff --git a/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala b/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala index 0037b7bc..c59f6e8b 100644 --- a/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala +++ b/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala @@ -234,114 +234,24 @@ class AdminHttpServerTest extends AnyFunSuite with Eventually with IntegrationPa } } - test("Default global configurator is empty") { - - var globalConfiguredServer: Option[Http.Server] = None + test("AdminServerInterface.Param for Admin server isAdmin=true") { class SpyServer extends TestTwitterServer { - var localConfiguredServer: Option[Http.Server] = None + var isAdminServer: Boolean = false override def configureAdminHttpServer(server: Http.Server): Http.Server = { - localConfiguredServer = Some( - super - .configureAdminHttpServer(server)) - localConfiguredServer.get + isAdminServer = server.params[AdminServerInterface.Param].isAdmin + server } } val server = new SpyServer { - override def main(): Unit = { - // not used - } - } - server.main(args = Array.empty[String]) - - assert(globalConfiguredServer == None) - assert(server.localConfiguredServer != None) - } - - test("Set global configurator gets run") { - - var globalConfiguredServer: Option[Http.Server] = None - - class SpyServer extends TestTwitterServer { - var localConfiguredServer: Option[Http.Server] = None - override def configureAdminHttpServer(server: Http.Server): Http.Server = { - localConfiguredServer = Some( - super - .configureAdminHttpServer(server)) - localConfiguredServer.get - } - } - - AdminHttpServer.setGlobalConfigurator { srv => - globalConfiguredServer = Some(srv) - srv - } - - val server = new SpyServer { override def main(): Unit = { // not used } } server.main(args = Array.empty[String]) - - assert(globalConfiguredServer == server.localConfiguredServer) - assert(server.localConfiguredServer != None) - } - - test("Global configurator throws and does not recover") { - - var globalConfiguredServer: Option[Http.Server] = None - - class SpyServer extends TestTwitterServer { - - private[this] var expectedThrowable: Option[Throwable] = None - - def setExpectedThrowable(t: Throwable) = { - expectedThrowable = Some(t) - } - - var localConfiguredServer: Option[Http.Server] = None - - override def configureAdminHttpServer(server: Http.Server): Http.Server = { - localConfiguredServer = Some( - super - .configureAdminHttpServer(server)) - localConfiguredServer.get - } - - override protected def exitOnError(throwable: Throwable): Unit = { - expectedThrowable match { - case Some(t) => assert(throwable == t) - case None => super.exitOnError(throwable) - } - } - } - - val badException = new Exception("Bad") - AdminHttpServer.setGlobalConfigurator { _ => - throw badException - } - - val server = new SpyServer { - override def main(): Unit = { - // not used - } - } - - server.setExpectedThrowable(badException) - try { - server.main(args = Array.empty[String]) - // server will have attempted to exit with error and asserted that error == expected - // verify state is as expected - assert(globalConfiguredServer == None) - assert(server.localConfiguredServer != None) - } finally { - // Reset here because this is global and can cause other tests to fail - AdminHttpServer.resetGlobalConfigurator() - } + assert(server.isAdminServer) } - } From 6599938f9b9779313e6e52362f7c325cca47bfe5 Mon Sep 17 00:00:00 2001 From: Anton Ivanov Date: Thu, 9 Jun 2022 14:31:31 +0000 Subject: [PATCH 360/409] twitter-server: Open AdminHttpServer start method to be able to start it manually # Problem AdminHttpServer is started during the premain phase which might be too early for /health checks. Unforunately an app has no means to start it manually later, because startServer method is private. # Solution Open startServer method for children. Rename it to startAdminHttpServer so it's clear what's started in case a child inherits several server-like traits. JIRA Issues: GRAPH-18693 Differential Revision: https://phabricator.twitter.biz/D910317 --- .../scala/com/twitter/server/AdminHttpServer.scala | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/AdminHttpServer.scala b/server/src/main/scala/com/twitter/server/AdminHttpServer.scala index 533fc5a0..ef3e3e70 100644 --- a/server/src/main/scala/com/twitter/server/AdminHttpServer.scala +++ b/server/src/main/scala/com/twitter/server/AdminHttpServer.scala @@ -317,7 +317,15 @@ trait AdminHttpServer { self: App with Stats => private[this] def routeToIndexLink(route: Route): IndexView.Link = IndexView.Link(route.alias, route.path, route.method) - private[this] def startServer(): Unit = { + /** + * Starts the server. + * + * By default the server starts automatically. + * If this is not desirable (e.g. to postpone replies from the /health endpoint) + * the server can be disabled with [[disableAdminHttpServer]]. + * In the latter case the server can still be started manually with this method. + */ + protected def startAdminHttpServer(): Unit = { val loggingMonitor = new Monitor { def handle(exc: Throwable): Boolean = { log.error(s"Caught exception in AdminHttpServer: $exc", exc) @@ -353,7 +361,7 @@ trait AdminHttpServer { self: App with Stats => if (disableAdminHttpServer) { log.info("admin http is disabled and will not be started.") } else { - startServer() + startAdminHttpServer() } } } From 78e5ff01e8d5471097def9e2ee320607d7fa5176 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Mon, 13 Jun 2022 16:46:29 +0000 Subject: [PATCH 361/409] finagle-core: Move SourceRole subtypes into its object Problem We have about a million things named `Client` or `Server` and that now includes the subtypes of `SourceRole`. This can make things really difficult to navigate. Solution Move the subtypes to the object of `SourceRole` to make the relationship more clear. Differential Revision: https://phabricator.twitter.biz/D909744 --- .../scala/com/twitter/server/util/exp/ExpressionJson.scala | 4 ++-- .../server/handler/MetricMetadataQueryHandlerTest.scala | 6 +++--- .../twitter/server/util/MetricSchemaJsonModuleTest.scala | 4 ++-- .../com/twitter/server/util/MetricSchemaSourceTest.scala | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/util/exp/ExpressionJson.scala b/server/src/main/scala/com/twitter/server/util/exp/ExpressionJson.scala index 4c6fcd2c..c00e864c 100644 --- a/server/src/main/scala/com/twitter/server/util/exp/ExpressionJson.scala +++ b/server/src/main/scala/com/twitter/server/util/exp/ExpressionJson.scala @@ -6,7 +6,7 @@ import com.fasterxml.jackson.databind.deser.std.StdDeserializer import com.fasterxml.jackson.databind.ser.std.StdSerializer import com.fasterxml.jackson.databind.DeserializationContext import com.fasterxml.jackson.databind.SerializerProvider -import com.twitter.finagle.stats.NoRoleSpecified +import com.twitter.finagle.stats.SourceRole import com.twitter.finagle.stats.exp._ /** @@ -35,7 +35,7 @@ object ExpressionJson { val labels = Map( ExpressionSchema.ProcessPath -> "Unspecified", ExpressionSchema.ServiceName -> "Unspecified", - ExpressionSchema.Role -> NoRoleSpecified.toString + ExpressionSchema.Role -> SourceRole.NoRoleSpecified.toString ) ++ expressionSchema.labels for ((key, value) <- labels) { gen.writeStringField(key, value) diff --git a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala index 1f7ba344..1bb1e1fb 100644 --- a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala @@ -21,7 +21,7 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { keyIndicator = true, description = "Counts how many cools are seen", units = Requests, - role = Server, + role = SourceRole.Server, verbosity = Verbosity.Default, sourceClass = Some("finagle.stats.cool"), name = Seq("my", "cool", "counter"), @@ -34,7 +34,7 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { keyIndicator = false, description = "Measures how fine the downstream system is", units = Percentage, - role = Client, + role = SourceRole.Client, verbosity = Verbosity.Debug, sourceClass = Some("finagle.stats.your"), name = Seq("your", "fine", "gauge"), @@ -53,7 +53,7 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { keyIndicator = true, description = "A counter scoped by null get deserialized correctly", units = Requests, - role = Server, + role = SourceRole.Server, verbosity = Verbosity.Default, sourceClass = Some("finagle.stats.bad"), name = Seq("my", "bad", null, "counter"), diff --git a/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala b/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala index a2a0a237..48dfd365 100644 --- a/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala +++ b/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala @@ -17,7 +17,7 @@ class MetricSchemaJsonModuleTest extends AnyFunSuite { keyIndicator = true, description = "Counts how many cools are seen", units = Requests, - role = Server, + role = SourceRole.Server, verbosity = Verbosity.Default, sourceClass = Some("finagle.stats.cool"), name = Seq("my", "cool", "counter"), @@ -30,7 +30,7 @@ class MetricSchemaJsonModuleTest extends AnyFunSuite { keyIndicator = false, description = "Measures how fine the downstream system is", units = Percentage, - role = Client, + role = SourceRole.Client, verbosity = Verbosity.Debug, sourceClass = Some("finagle.stats.your"), name = Seq("your", "fine", "gauge"), diff --git a/server/src/test/scala/com/twitter/server/util/MetricSchemaSourceTest.scala b/server/src/test/scala/com/twitter/server/util/MetricSchemaSourceTest.scala index e5ce39cc..20b74cdf 100644 --- a/server/src/test/scala/com/twitter/server/util/MetricSchemaSourceTest.scala +++ b/server/src/test/scala/com/twitter/server/util/MetricSchemaSourceTest.scala @@ -16,7 +16,7 @@ class MetricSchemaSourceTest extends AnyFunSuite { keyIndicator = true, description = "Counts how many cools are seen", units = Requests, - role = Server, + role = SourceRole.Server, verbosity = Verbosity.Default, sourceClass = Some("finagle.stats.cool"), name = Seq("my", "cool", "counter"), @@ -29,7 +29,7 @@ class MetricSchemaSourceTest extends AnyFunSuite { keyIndicator = false, description = "Measures how fine the downstream system is", units = Percentage, - role = Client, + role = SourceRole.Client, verbosity = Verbosity.Debug, sourceClass = Some("finagle.stats.your"), name = Seq("your", "fine", "gauge"), From 6bc9a7241b88c452a1189cdf77589a36da67512d Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Tue, 21 Jun 2022 21:25:48 +0000 Subject: [PATCH 362/409] finagle-core: Don't migrate all metrics yet Problem We are not sure we want to migrate users metrics to dimensional form yet, but we do want to start working on finagles metrics. Solution Make metrics hierarchical only by default but turn all finagle metrics into dimensional metrics as well via in a special proxy StatsReceiver. Differential Revision: https://phabricator.twitter.biz/D909544 --- .../MetricMetadataQueryHandlerTest.scala | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala index 1bb1e1fb..8e8ba4df 100644 --- a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala @@ -4,6 +4,7 @@ import com.twitter.finagle.http.Request import com.twitter.finagle.stats.MetricBuilder.CounterType import com.twitter.finagle.stats.MetricBuilder.GaugeType import com.twitter.finagle.stats.MetricBuilder.HistogramType +import com.twitter.finagle.stats.MetricBuilder.Identity import com.twitter.finagle.stats._ import com.twitter.finagle.stats.exp.ExpressionSchema import com.twitter.finagle.stats.exp.ExpressionSchemaKey @@ -28,7 +29,13 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { processPath = Some("dc/role/zone/service"), percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), metricType = CounterType - ).withLabels(Map("label1" -> "value1", "label2" -> "value2")), + ).withIdentity( + Identity( + Seq("my", "cool", "counter"), + Seq("my", "cool", "counter"), + Map("label1" -> "value1", "label2" -> "value2"), + hierarchicalOnly = false + )), "your/fine/gauge" -> MetricBuilder( keyIndicator = false, @@ -44,10 +51,15 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { ), "my/only/histo" -> MetricBuilder( - name = Seq("my", "only", "histo"), percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), metricType = HistogramType - ).withLabels(labels = Map("label1" -> "value1")), + ).withIdentity( + Identity( + Seq("my", "only", "histo"), + Seq("my", "only", "histo"), + Map("label1" -> "value1"), + hierarchicalOnly = false, + )), "my/bad/null/counter" -> MetricBuilder( keyIndicator = true, @@ -134,7 +146,7 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { | "dimensional_name" : "your_fine_gauge", | "relative_name" : ["your","fine","gauge"], | "labels" : {}, - | "dimensional_support" : true, + | "dimensional_support" : false, | "kind" : "gauge", | "source" : { | "class": "finagle.stats.your", @@ -183,7 +195,7 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { | "dimensional_name" : "my_bad_null_counter", | "relative_name" : ["my","bad", "null", "counter"], | "labels" : {}, - | "dimensional_support" : true, + | "dimensional_support" : false, | "kind" : "counter", | "source" : { | "class": "finagle.stats.bad", @@ -207,7 +219,7 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { | "dimensional_name" : "your_fine_gauge", | "relative_name" : ["your","fine","gauge"], | "labels" : {}, - | "dimensional_support" : true, + | "dimensional_support" : false, | "kind" : "gauge", | "source" : { | "class": "finagle.stats.your", @@ -250,7 +262,7 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { | "dimensional_name" : "your_fine_gauge", | "relative_name" : ["your","fine","gauge"], | "labels" : {}, - | "dimensional_support" : true, + | "dimensional_support" : false, | "kind" : "gauge", | "source" : { | "class": "finagle.stats.your", From 07f1c768147d884576843c0617e702bd8b5db7bf Mon Sep 17 00:00:00 2001 From: Nora Howard Date: Fri, 1 Jul 2022 02:41:59 +0000 Subject: [PATCH 363/409] [jdk-11-migration] mark targets that must stay on java8 for one scalding target Explicitly mark targets that cannot migrate to java11 as java8, so they will not be affected when the default platform is changed to java11. Targets that cannot migrate include hadoop and scalding jobs, targets that they depend on, and other special cases. See https//go.twitter.biz/jdk11migration for detailed migration docs. This patch is for src/scala/com/twitter/ads/batch/job/targeting/handle_ml/trainer:handle_targeting_training_data_gen-binary and its dependencies. JIRA Issues: VM-5402 Differential Revision: https://phabricator.twitter.biz/D920109 --- server/src/main/scala/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/scala/BUILD b/server/src/main/scala/BUILD index 76b446b5..a51834f8 100644 --- a/server/src/main/scala/BUILD +++ b/server/src/main/scala/BUILD @@ -1,6 +1,7 @@ scala_library( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "twitter-server", From d7ac428319016033579e60b1481fcee162c52e25 Mon Sep 17 00:00:00 2001 From: Uriel Salazar Date: Fri, 8 Jul 2022 21:52:30 +0000 Subject: [PATCH 364/409] VM-5413: batch exclude heron binary targets from migration by marking them with platform 8 Explicitly mark targets that cannot migrate to java11 as java8, so they will not be affected when the default platform is changed to java11. Targets that cannot migrate include hadoop and scalding jobs, targets that they depend on, and other special cases. See https//go.twitter.biz/jdk11migration for detailed migration docs. This patch marks all of the heron_binarary targets and their dependencies. JIRA Issues: VM-5415 Differential Revision: https://phabricator.twitter.biz/D925912 --- logback-classic/src/main/scala/BUILD | 1 + slf4j-jdk14/src/main/scala/BUILD | 1 + slf4j-jdk14/src/main/scala/com/twitter/server/logging/BUILD | 1 + 3 files changed, 3 insertions(+) diff --git a/logback-classic/src/main/scala/BUILD b/logback-classic/src/main/scala/BUILD index 1c273173..8dd9c766 100644 --- a/logback-classic/src/main/scala/BUILD +++ b/logback-classic/src/main/scala/BUILD @@ -1,6 +1,7 @@ scala_library( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "twitter-server-logback-classic", diff --git a/slf4j-jdk14/src/main/scala/BUILD b/slf4j-jdk14/src/main/scala/BUILD index 2997bd1d..297ed41d 100644 --- a/slf4j-jdk14/src/main/scala/BUILD +++ b/slf4j-jdk14/src/main/scala/BUILD @@ -1,6 +1,7 @@ scala_library( sources = ["com/twitter/server/handler/slf4j/jdk14/*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "twitter-server-slf4j-jdk14", diff --git a/slf4j-jdk14/src/main/scala/com/twitter/server/logging/BUILD b/slf4j-jdk14/src/main/scala/com/twitter/server/logging/BUILD index aa6d6fdb..5c4b7eae 100644 --- a/slf4j-jdk14/src/main/scala/com/twitter/server/logging/BUILD +++ b/slf4j-jdk14/src/main/scala/com/twitter/server/logging/BUILD @@ -1,6 +1,7 @@ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "twitter-server-slf4j-jdk14-logging", From c78725ea8b8fec23979e3ed3496cea3a9372a502 Mon Sep 17 00:00:00 2001 From: Nora Howard Date: Thu, 14 Jul 2022 09:48:54 +0000 Subject: [PATCH 365/409] VM-5413: batch exclude scalding_job targets and their dependencies Explicitly mark targets that cannot migrate to java11 as java8, so they will not be affected when the default platform is changed to java11. Targets that cannot migrate include hadoop and scalding jobs, targets that they depend on, and other special cases. See https//go.twitter.biz/jdk11migration for detailed migration docs. This patch marks all of the scalding_job targets and their dependencies. JIRA Issues: VM-5413 Differential Revision: https://phabricator.twitter.biz/D925117 --- opencensus/src/main/scala/com/twitter/server/opencensus/BUILD | 1 + slf4j-jdk14/src/main/scala/com/twitter/server/slf4j/jdk14/BUILD | 1 + 2 files changed, 2 insertions(+) diff --git a/opencensus/src/main/scala/com/twitter/server/opencensus/BUILD b/opencensus/src/main/scala/com/twitter/server/opencensus/BUILD index 929d1311..bc0920df 100644 --- a/opencensus/src/main/scala/com/twitter/server/opencensus/BUILD +++ b/opencensus/src/main/scala/com/twitter/server/opencensus/BUILD @@ -1,6 +1,7 @@ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/io/opencensus:opencensus-api", diff --git a/slf4j-jdk14/src/main/scala/com/twitter/server/slf4j/jdk14/BUILD b/slf4j-jdk14/src/main/scala/com/twitter/server/slf4j/jdk14/BUILD index 82ccc357..9e263913 100644 --- a/slf4j-jdk14/src/main/scala/com/twitter/server/slf4j/jdk14/BUILD +++ b/slf4j-jdk14/src/main/scala/com/twitter/server/slf4j/jdk14/BUILD @@ -1,6 +1,7 @@ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "twitter-server-slf4j-jdk14", From 5903ab6e63e94dcecce2f98a96935bce63a9f988 Mon Sep 17 00:00:00 2001 From: Nora Howard Date: Thu, 14 Jul 2022 20:18:53 +0000 Subject: [PATCH 366/409] [jdk-11-migration] batch mark hadoop_binary targets and dependencies to use java8 Explicitly mark targets that cannot migrate to java11 as java8, so they will not be affected when the default platform is changed to java11. Targets that cannot migrate include hadoop and scalding jobs, targets that they depend on, and other special cases. See https//go.twitter.biz/jdk11migration for detailed migration docs. This patch marks all of the hadoop_binary targets and their dependencies. JIRA Issues: VM-5414 Differential Revision: https://phabricator.twitter.biz/D927188 --- slf4j-log4j12/src/main/scala/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/slf4j-log4j12/src/main/scala/BUILD b/slf4j-log4j12/src/main/scala/BUILD index 5750ad3d..97a6d449 100644 --- a/slf4j-log4j12/src/main/scala/BUILD +++ b/slf4j-log4j12/src/main/scala/BUILD @@ -1,6 +1,7 @@ scala_library( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "twitter-server-slf4j-log4j12", From f682d631f52d99c6df4d6a1ef2b307779b267323 Mon Sep 17 00:00:00 2001 From: Anthony Madrigal Date: Wed, 20 Jul 2022 18:12:28 +0000 Subject: [PATCH 367/409] VM-5465:updated BUILD files JIRA Issues: VM-5465 Differential Revision: https://phabricator.twitter.biz/D936033 --- logback-classic/src/test/scala/BUILD | 1 + server/src/test/scala/BUILD | 1 + slf4j-jdk14/src/test/scala/BUILD | 1 + slf4j-log4j12/src/test/scala/BUILD | 1 + 4 files changed, 4 insertions(+) diff --git a/logback-classic/src/test/scala/BUILD b/logback-classic/src/test/scala/BUILD index 638ae174..925a8eab 100644 --- a/logback-classic/src/test/scala/BUILD +++ b/logback-classic/src/test/scala/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/ch/qos/logback:logback-classic", diff --git a/server/src/test/scala/BUILD b/server/src/test/scala/BUILD index cd1e2ab4..7bf616f2 100644 --- a/server/src/test/scala/BUILD +++ b/server/src/test/scala/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", diff --git a/slf4j-jdk14/src/test/scala/BUILD b/slf4j-jdk14/src/test/scala/BUILD index 57357959..ecb6d2f1 100644 --- a/slf4j-jdk14/src/test/scala/BUILD +++ b/slf4j-jdk14/src/test/scala/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/junit", diff --git a/slf4j-log4j12/src/test/scala/BUILD b/slf4j-log4j12/src/test/scala/BUILD index af2d3975..087d6dd9 100644 --- a/slf4j-log4j12/src/test/scala/BUILD +++ b/slf4j-log4j12/src/test/scala/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/junit", From 25e4a30918260c7d3d9c5d68f49106cb01aa7f78 Mon Sep 17 00:00:00 2001 From: Alicia Vargas-Morawetz Date: Tue, 26 Jul 2022 17:10:27 +0000 Subject: [PATCH 368/409] [util,scrooge,twitter-server,finagle,finatra]: Upgrade sbt to 1.7.1 Problem / Solution Upgrade SBT to 1.7.1 for CSL libraries and upgrade some plugins as well. JIRA Issues: CSL-12170 Differential Revision: https://phabricator.twitter.biz/D938280 --- .github/workflows/continuous-integration.yml | 2 +- project/build.properties | 2 +- project/plugins.sbt | 4 ++-- sbt | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 1647a6ea..a79149ea 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -3,7 +3,7 @@ name: continuous integration env: JAVA_OPTS: "-Dsbt.log.noformat=true" TRAVIS: "true" # pretend we're TravisCI - SBT_VERSION: 1.6.1 + SBT_VERSION: 1.7.1 BUILD_VERSION: "v1" # bump this if builds are failing due to a bad cache defaults: diff --git a/project/build.properties b/project/build.properties index 3161d214..22af2628 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.6.1 +sbt.version=1.7.1 diff --git a/project/plugins.sbt b/project/plugins.sbt index e55bc980..67d03eba 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -4,5 +4,5 @@ resolvers ++= Seq( ) addSbtPlugin("com.typesafe.sbt" % "sbt-site" % "1.4.1") -addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.9.1") -addSbtPlugin("com.eed3si9n" % "sbt-unidoc" % "0.4.3") +addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.0.0") +addSbtPlugin("com.github.sbt" % "sbt-unidoc" % "0.5.0") diff --git a/sbt b/sbt index 1d62b302..d08998b1 100755 --- a/sbt +++ b/sbt @@ -2,9 +2,9 @@ set -eo pipefail -sbtver="1.6.1" +sbtver="1.7.1" sbtjar="sbt-launch-$sbtver.jar" -sbtsha128="b69ab7f114d09d859b5e349acdb05d32c9d49072" +sbtsha128="468efdd45baf58dbb575f9b4369c5234f8cd54ba" sbtrepo="https://repo1.maven.org/maven2/org/scala-sbt/sbt-launch" From 6b9bf60fbc8df03c9b56bba46dd2ad074a67024e Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Tue, 26 Jul 2022 21:27:19 +0000 Subject: [PATCH 369/409] twitter-server: Use default HTTP implementation (HTTP2) for admin server Problem TwitterServer's admin HTTP server was forcing HTTP/1.1. Solution Use Finagle's default HTTP protocol version (happens to be HTTP/2 at the moment). JIRA Issues: CSL-12117 Differential Revision: https://phabricator.twitter.biz/D920963 --- CHANGELOG.rst | 4 ++++ .../src/main/scala/com/twitter/server/AdminHttpServer.scala | 1 - .../test/scala/com/twitter/server/TwitterServerTest.scala | 5 +++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 44e762d0..b3d9d3ba 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,10 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +* Admin HTTP server is now on Finagle's default HTTP protocol version, which supports + HTTP/1.1, HTTP/2 cleartext protocol upgrade, HTTP/2 prior-knowledge, and ALPN. + ``PHAB_ID=D920963`` + 22.4.0 ------ diff --git a/server/src/main/scala/com/twitter/server/AdminHttpServer.scala b/server/src/main/scala/com/twitter/server/AdminHttpServer.scala index ef3e3e70..0b9f69a3 100644 --- a/server/src/main/scala/com/twitter/server/AdminHttpServer.scala +++ b/server/src/main/scala/com/twitter/server/AdminHttpServer.scala @@ -336,7 +336,6 @@ trait AdminHttpServer { self: App with Stats => log.info(s"Serving admin http on ${adminPort()}") adminHttpServer = configureAdminHttpServer( Http.server - .configured(Http.Netty4Impl) .withStatsReceiver(NullStatsReceiver) .withTracer(NullTracer) .withMonitor(loggingMonitor) diff --git a/server/src/test/scala/com/twitter/server/TwitterServerTest.scala b/server/src/test/scala/com/twitter/server/TwitterServerTest.scala index b79ac06c..8eaf62de 100644 --- a/server/src/test/scala/com/twitter/server/TwitterServerTest.scala +++ b/server/src/test/scala/com/twitter/server/TwitterServerTest.scala @@ -3,7 +3,8 @@ package com.twitter.server import com.twitter.concurrent.NamedPoolThreadFactory import com.twitter.conversions.DurationOps._ import com.twitter.finagle.Service -import com.twitter.finagle.http.{Request, Response} +import com.twitter.finagle.http.Request +import com.twitter.finagle.http.Response import com.twitter.util._ import java.util.concurrent.Executors import scala.collection.mutable @@ -13,7 +14,7 @@ import org.scalatest.funsuite.AnyFunSuite class TestTwitterServer(await: Boolean = false) extends TwitterServer { override val defaultAdminPort = 0 /* ensure enough time to close resources */ - override val defaultCloseGracePeriod: Duration = 30.seconds + override val defaultCloseGracePeriod: Duration = 1.second val bootstrapSeq: mutable.ArrayBuffer[Symbol] = mutable.ArrayBuffer.empty[Symbol] val value: Promise[Unit] = new Promise[Unit] From b30d84e0e4087354b5879b88716c395c2cfb5589 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Tue, 26 Jul 2022 23:05:30 +0000 Subject: [PATCH 370/409] Twitter-oss: Prepare OSS libraries for release 22.7 Problem We want to release the next versions of our Twitter OSS libraries 22.7 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-12182 Differential Revision: https://phabricator.twitter.biz/D942680 --- CHANGELOG.rst | 3 +++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b3d9d3ba..e1615aa1 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +22.7.0 +------ + * Admin HTTP server is now on Finagle's default HTTP protocol version, which supports HTTP/1.1, HTTP/2 cleartext protocol upgrade, HTTP/2 prior-knowledge, and ALPN. ``PHAB_ID=D920963`` diff --git a/README.md b/README.md index c4c5dc37..320da6d9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index b11323da..b1153141 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "22.5.0-SNAPSHOT" +val releaseVersion = "22.7.0" val logbackVersion = "1.2.8" val opencensusVersion = "0.19.1" From 08a0edf9f7d0105175487524a70d465a176f3871 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Wed, 27 Jul 2022 02:36:08 +0000 Subject: [PATCH 371/409] Twitter-oss: update OSS libraries post-release to 22.10.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 22.10.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D942842 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 320da6d9..c4c5dc37 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index b1153141..ccc2693b 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "22.7.0" +val releaseVersion = "22.10.0-SNAPSHOT" val logbackVersion = "1.2.8" val opencensusVersion = "0.19.1" From 5817ead04ebe1ef4d0f7ab61fcc7b89d7de8c794 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Thu, 28 Jul 2022 16:34:57 +0000 Subject: [PATCH 372/409] Revert "Twitter-oss: update OSS libraries post-release to 22.10.0-SNAPSHOT" This reverts commit 7915f823ec292ddd448489b79935691dcad756bf. Differential Revision: https://phabricator.twitter.biz/D943947 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c4c5dc37..320da6d9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index ccc2693b..b1153141 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "22.10.0-SNAPSHOT" +val releaseVersion = "22.7.0" val logbackVersion = "1.2.8" val opencensusVersion = "0.19.1" From 8d81ab28bcd538274ba5ee5c95bc2146ab202b4c Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Thu, 28 Jul 2022 16:38:43 +0000 Subject: [PATCH 373/409] Downgrade Scoverage plugin version Problem There is a conflict between scalatest and scoverage with the current version combination that is causing some of the builds to fail. Solution Let's just downgrade scoverage until we get a better solution. JIRA Issues: CSL-12207 Differential Revision: https://phabricator.twitter.biz/D943932 --- project/plugins.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/plugins.sbt b/project/plugins.sbt index 67d03eba..a162e5c2 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -4,5 +4,5 @@ resolvers ++= Seq( ) addSbtPlugin("com.typesafe.sbt" % "sbt-site" % "1.4.1") -addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.0.0") +addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.9.1") addSbtPlugin("com.github.sbt" % "sbt-unidoc" % "0.5.0") From 612fbed57b13c1dea76d3e3fcdd08e35a4a5cc68 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Thu, 28 Jul 2022 17:58:45 +0000 Subject: [PATCH 374/409] Twitter-oss: update OSS libraries post-release to 22.10.0-SNAPSHOT (revert of a revert) This reverts commit 04ecaac33a1bd2ddb9eede8e8264ede5af3fd161. Differential Revision: https://phabricator.twitter.biz/D944382 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 320da6d9..c4c5dc37 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index b1153141..ccc2693b 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "22.7.0" +val releaseVersion = "22.10.0-SNAPSHOT" val logbackVersion = "1.2.8" val opencensusVersion = "0.19.1" From 75cf18cb605068efdc9adf2fb48211a1f1ec5dd1 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Fri, 29 Jul 2022 18:38:40 +0000 Subject: [PATCH 375/409] Remove scoverage integration Problem We've run into more problems with scoverage integration. Solution Remove scoverage. It's a cool feature but effectively never utilized anyway, resulting in all pain no gain. JIRA Issues: CSL-12211 Differential Revision: https://phabricator.twitter.biz/D945589 --- .github/workflows/continuous-integration.yml | 6 +----- README.md | 1 - build.sbt | 3 --- project/plugins.sbt | 1 - 4 files changed, 1 insertion(+), 10 deletions(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index a79149ea..7d8815ef 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -61,8 +61,4 @@ jobs: publish-m2: true verbose: true - name: test - run: ${{ format('./sbt ++{0} clean coverage test coverageReport', matrix.scala) }} - - name: coverage - run: ${{ format('./sbt ++{0} coverageAggregate', matrix.scala) }} - - name: upload coverage - uses: codecov/codecov-action@v2 + run: ${{ format('./sbt ++{0} clean test', matrix.scala) }} diff --git a/README.md b/README.md index c4c5dc37..39a9998d 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@ # TwitterServer [![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) -[![Codecov](https://codecov.io/gh/twitter/twitter-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/twitter-server) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) [![Maven Central](https://maven-badges.herokuapp.com/maven-central/com.twitter/twitter-server_2.12/badge.svg)](https://maven-badges.herokuapp.com/maven-central/com.twitter/twitter-server_2.12) diff --git a/build.sbt b/build.sbt index ccc2693b..f8502e69 100644 --- a/build.sbt +++ b/build.sbt @@ -1,5 +1,3 @@ -import scoverage.ScoverageKeys - Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions @@ -87,7 +85,6 @@ lazy val sharedSettings = Seq( "org.scalatestplus" %% "mockito-1-10" % "3.1.0.0" % "test", "org.scalatestplus" %% "scalacheck-1-14" % "3.1.2.0" % "test" ), - ScoverageKeys.coverageHighlighting := true, ivyXML := diff --git a/project/plugins.sbt b/project/plugins.sbt index a162e5c2..8be80fda 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -4,5 +4,4 @@ resolvers ++= Seq( ) addSbtPlugin("com.typesafe.sbt" % "sbt-site" % "1.4.1") -addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.9.1") addSbtPlugin("com.github.sbt" % "sbt-unidoc" % "0.5.0") From 0fcff2b58ad1b2dd920998ba9d63a916b2ab9f3a Mon Sep 17 00:00:00 2001 From: Benjamin Mahler Date: Tue, 9 Aug 2022 23:07:20 +0000 Subject: [PATCH 376/409] Lint Lifecycle.scala. Problem I was touching this file to add a message to the warmup /health response, and was getting linting noise in my diff. Solution This addresses the linting issues in a clean patch. Result This file passes the linter. Other changes to the file don't need to include noisy linting changes. Differential Revision: https://phabricator.twitter.biz/D949332 --- server/src/main/scala/com/twitter/server/Lifecycle.scala | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/Lifecycle.scala b/server/src/main/scala/com/twitter/server/Lifecycle.scala index 9da5a62f..cadfdb25 100644 --- a/server/src/main/scala/com/twitter/server/Lifecycle.scala +++ b/server/src/main/scala/com/twitter/server/Lifecycle.scala @@ -2,9 +2,12 @@ package com.twitter.server import com.twitter.app.GlobalFlag import com.twitter.app.lifecycle.Event -import com.twitter.app.lifecycle.Event.{PrebindWarmup, WarmupComplete} +import com.twitter.app.lifecycle.Event.PrebindWarmup +import com.twitter.app.lifecycle.Event.WarmupComplete import com.twitter.finagle.http.Method.Post -import com.twitter.finagle.http.{HttpMuxer, Route, RouteIndex} +import com.twitter.finagle.http.HttpMuxer +import com.twitter.finagle.http.Route +import com.twitter.finagle.http.RouteIndex import com.twitter.server.handler._ import java.lang.management.ManagementFactory import java.util.concurrent.atomic.AtomicBoolean From 34ec026bacb22f0bbba87a8296ee0dcb67fce3b1 Mon Sep 17 00:00:00 2001 From: Benjamin Mahler Date: Thu, 11 Aug 2022 21:02:08 +0000 Subject: [PATCH 377/409] Add a more informative health message while warming up. Problem Currently, between warmup initialization and completion, the /health handler responds with an empty string rather than `"OK"`, which isn't very helpful to indicate why the application isn't healthy. Solution This adds a more informative message to the /health response when the warmup is still in progress. Result Now when health checks fail during warmup, users and operators will see something like 'Failed health check! Response differs from expected response (expected "ok", got "warming up")'. Differential Revision: https://phabricator.twitter.biz/D949333 --- CHANGELOG.rst | 3 +++ server/src/main/scala/com/twitter/server/Lifecycle.scala | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e1615aa1..ca596abc 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +* During warmup, the /health route will respond with a more informative "warming up" + message, previously an empty message was used. ``PHAB_ID=D949333`` + 22.7.0 ------ diff --git a/server/src/main/scala/com/twitter/server/Lifecycle.scala b/server/src/main/scala/com/twitter/server/Lifecycle.scala index cadfdb25..bc1ac849 100644 --- a/server/src/main/scala/com/twitter/server/Lifecycle.scala +++ b/server/src/main/scala/com/twitter/server/Lifecycle.scala @@ -87,7 +87,7 @@ object Lifecycle { * Initialize warmup code. Ensures that the /health endpoint will not return on "OK" response. */ def initializeWarmup(): Unit = - HttpMuxer.addHandler(Route("/health", new ReplyHandler(""))) + HttpMuxer.addHandler(Route("/health", new ReplyHandler("warming up\n"))) /** * Prebind warmup code. Used for warmup tasks that we want to run before we From 2ae1c9781414ecb4b41c5ed16b9080b5bd3ec800 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Mon, 15 Aug 2022 20:29:01 +0000 Subject: [PATCH 378/409] util-stats, twitter-server: Add a ShortLived verbosity level Problem Some metrics are not valuable for a long time and it would be nice if the backend metrics store could drop them early to avoid the cost of longer term storage. Solution Add a ShortLived metrics verbosity level that can signal to the metrics store through the metric metadata endpoint that this metrics is of limited value for long duration. Differential Revision: https://phabricator.twitter.biz/D954203 --- .../handler/MetricMetadataQueryHandler.scala | 6 +++--- .../MetricMetadataQueryHandlerTest.scala | 20 +++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/handler/MetricMetadataQueryHandler.scala b/server/src/main/scala/com/twitter/server/handler/MetricMetadataQueryHandler.scala index 1e796165..750fdeab 100644 --- a/server/src/main/scala/com/twitter/server/handler/MetricMetadataQueryHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/MetricMetadataQueryHandler.scala @@ -41,7 +41,7 @@ import com.twitter.util.Future * }, * "description" : "Counts how many cools are seen", * "unit" : "Requests", - * "verbosity": "Verbosity(default)", + * "verbosity": "default", * "key_indicator" : true * }, * { @@ -55,7 +55,7 @@ import com.twitter.util.Future * }, * "description" : "Measures how fine the downstream system is", * "unit" : "Percentage", - * "verbosity": "Verbosity(debug)", + * "verbosity": "debug", * "key_indicator" : false * }, * { @@ -69,7 +69,7 @@ import com.twitter.util.Future * }, * "description" : "No description provided", * "unit" : "Unspecified", - * "verbosity": "Verbosity(default)", + * "verbosity": "default", * "key_indicator" : false, * "buckets" : { * "count" : ".count", diff --git a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala index 8e8ba4df..4601cd69 100644 --- a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala @@ -138,7 +138,7 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { | }, | "description" : "Counts how many cools are seen", | "unit" : "Requests", - | "verbosity": "Verbosity(default)", + | "verbosity": "default", | "key_indicator" : true | }, | { @@ -155,7 +155,7 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { | }, | "description" : "Measures how fine the downstream system is", | "unit" : "Percentage", - | "verbosity": "Verbosity(debug)", + | "verbosity": "debug", | "key_indicator" : false | }, | { @@ -174,7 +174,7 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { | }, | "description" : "No description provided", | "unit" : "Unspecified", - | "verbosity": "Verbosity(default)", + | "verbosity": "default", | "key_indicator" : false, | "buckets" : { | "count" : ".count", @@ -204,7 +204,7 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { | }, | "description" : "A counter scoped by null get deserialized correctly", | "unit" : "Requests", - | "verbosity": "Verbosity(default)", + | "verbosity": "default", | "key_indicator" : true | } | ] @@ -228,7 +228,7 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { | }, | "description" : "Measures how fine the downstream system is", | "unit" : "Percentage", - | "verbosity": "Verbosity(debug)", + | "verbosity": "debug", | "key_indicator" : false | } | ] @@ -254,7 +254,7 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { | }, | "description" : "Counts how many cools are seen", | "unit" : "Requests", - | "verbosity": "Verbosity(default)", + | "verbosity": "default", | "key_indicator" : true | }, | { @@ -271,7 +271,7 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { | }, | "description" : "Measures how fine the downstream system is", | "unit" : "Percentage", - | "verbosity": "Verbosity(debug)", + | "verbosity": "debug", | "key_indicator" : false | } | ] @@ -297,7 +297,7 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { | }, | "description" : "No description provided", | "unit" : "Unspecified", - | "verbosity": "Verbosity(default)", + | "verbosity": "default", | "key_indicator" : false, | "buckets" : { | "count" : ".count", @@ -336,7 +336,7 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { | }, | "description" : "Counts how many cools are seen", | "unit" : "Requests", - | "verbosity": "Verbosity(default)", + | "verbosity": "default", | "key_indicator" : true | }, | { @@ -355,7 +355,7 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { | }, | "description" : "No description provided", | "unit" : "Unspecified", - | "verbosity": "Verbosity(default)", + | "verbosity": "default", | "key_indicator" : false, | "buckets" : { | "count" : ".count", From d821eb09647d858a0f17cb2e175e7adc51b355a8 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Thu, 18 Aug 2022 07:13:53 +0000 Subject: [PATCH 379/409] twitter-server: Export histogram_format metadata Problem / Solution Histograms now have a new configuration option - `HistogramFormat`. Let's export it in the metadata. JIRA Issues: CSL-12237 Differential Revision: https://phabricator.twitter.biz/D955933 --- .../com/twitter/server/util/MetricSchemaSerializer.scala | 1 + .../server/handler/MetricMetadataQueryHandlerTest.scala | 6 +++++- .../twitter/server/util/MetricSchemaJsonModuleTest.scala | 8 +++++--- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala b/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala index 8b1c9e35..1d6fcd7c 100644 --- a/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala +++ b/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala @@ -86,6 +86,7 @@ object SchemaSerializer extends StdSerializer[MetricBuilder](classOf[MetricBuild metricBuilder.metricType match { case HistogramType => + jsonGenerator.writeStringField("histogram_format", metricBuilder.histogramFormat.toString) jsonGenerator.writeObjectFieldStart("buckets") jsonGenerator.writeStringField("count", statsFormatter.histogramSeparator + "count") jsonGenerator.writeStringField("sum", statsFormatter.histogramSeparator + "sum") diff --git a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala index 4601cd69..7f4d8b91 100644 --- a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala @@ -52,7 +52,8 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { "my/only/histo" -> MetricBuilder( percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), - metricType = HistogramType + metricType = HistogramType, + histogramFormat = HistogramFormat.ShortSummary, ).withIdentity( Identity( Seq("my", "only", "histo"), @@ -176,6 +177,7 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { | "unit" : "Unspecified", | "verbosity": "default", | "key_indicator" : false, + | "histogram_format": "short_summary", | "buckets" : { | "count" : ".count", | "sum" : ".sum", @@ -299,6 +301,7 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { | "unit" : "Unspecified", | "verbosity": "default", | "key_indicator" : false, + | "histogram_format": "short_summary", | "buckets" : { | "count" : ".count", | "sum" : ".sum", @@ -357,6 +360,7 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { | "unit" : "Unspecified", | "verbosity": "default", | "key_indicator" : false, + | "histogram_format": "short_summary", | "buckets" : { | "count" : ".count", | "sum" : ".sum", diff --git a/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala b/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala index 48dfd365..a32797d7 100644 --- a/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala +++ b/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala @@ -42,7 +42,8 @@ class MetricSchemaJsonModuleTest extends AnyFunSuite { MetricBuilder( name = Seq("my", "only", "histo"), percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), - metricType = HistogramType + metricType = HistogramType, + histogramFormat = HistogramFormat.ShortSummary ) private val topLevelFieldNameSet = @@ -90,16 +91,17 @@ class MetricSchemaJsonModuleTest extends AnyFunSuite { } test( - "HistogramSchema serializes with kind histogram and the correct set of fields (includeing buckets)") { + "HistogramSchema serializes with kind histogram and the correct set of fields (including buckets)") { val serializedString = AdminJsonConverter.writeToString(histogramSchema) val jsonMap = jsonStrToMap(serializedString) - assert(jsonMap.keys == topLevelFieldNameSet ++ Seq("buckets")) + assert(jsonMap.keys == topLevelFieldNameSet ++ Seq("histogram_format", "buckets")) assert( jsonMap.get("source").get == Map( "class" -> "Unspecified", "category" -> "NoRoleSpecified", "process_path" -> "Unspecified")) assert(jsonMap.get("kind").get == "histogram") + assert(jsonMap.get("histogram_format").get == "short_summary") assert( jsonMap.get("buckets").get == Map( "count" -> ".count", From 9a2b5df58dda0a7ba9055d25774021b95cad90b1 Mon Sep 17 00:00:00 2001 From: Jared Crawford Date: Fri, 26 Aug 2022 02:53:05 +0000 Subject: [PATCH 380/409] [Bazel migration] migrate 0 dependee test targets to bazel only JIRA Issues: PSEC-13747 TBR=true Differential Revision: https://phabricator.twitter.biz/D961872 --- logback-classic/src/test/scala/{BUILD => BUILD.bazel} | 0 slf4j-log4j12/src/test/scala/{BUILD => BUILD.bazel} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename logback-classic/src/test/scala/{BUILD => BUILD.bazel} (100%) rename slf4j-log4j12/src/test/scala/{BUILD => BUILD.bazel} (100%) diff --git a/logback-classic/src/test/scala/BUILD b/logback-classic/src/test/scala/BUILD.bazel similarity index 100% rename from logback-classic/src/test/scala/BUILD rename to logback-classic/src/test/scala/BUILD.bazel diff --git a/slf4j-log4j12/src/test/scala/BUILD b/slf4j-log4j12/src/test/scala/BUILD.bazel similarity index 100% rename from slf4j-log4j12/src/test/scala/BUILD rename to slf4j-log4j12/src/test/scala/BUILD.bazel From 42d307ed423014df39c7928a52ede05284e08b2a Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Mon, 29 Aug 2022 21:25:05 +0000 Subject: [PATCH 381/409] finagle-core: Remove the PanicMode toggle Problem We are finished with the PanicMode rollout but the toggle remains. Solution Remove the toggle and rectify the tests to account for the new default. Differential Revision: https://phabricator.twitter.biz/D966246 --- .../scala/com/twitter/server/view/BalancersJsonViewTest.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/test/scala/com/twitter/server/view/BalancersJsonViewTest.scala b/server/src/test/scala/com/twitter/server/view/BalancersJsonViewTest.scala index 8fbf077b..4c25d379 100644 --- a/server/src/test/scala/com/twitter/server/view/BalancersJsonViewTest.scala +++ b/server/src/test/scala/com/twitter/server/view/BalancersJsonViewTest.scala @@ -25,7 +25,7 @@ class BalancersJsonViewTest extends AnyFunSuite { val registry = BalancerRegistry.get.allMetadata.filter(_.label == label) assert(registry.size == 1) - val json = (new BalancersJsonView(registry)).render + val json = new BalancersJsonView(registry).render val expected = s"""{ | "clients" : [ @@ -40,7 +40,7 @@ class BalancersJsonViewTest extends AnyFunSuite { | "total_pending" : 0, | "total_load" : 0.0, | "size" : 1, - | "panic_mode" : "MajorityUnhealthy", + | "panic_mode" : "FiftyPercentUnhealthy", | "additional_info" : { } | } | } From bd51073ac08ca5c6389f913296403757a0911043 Mon Sep 17 00:00:00 2001 From: Eitan Adler Date: Wed, 31 Aug 2022 06:10:59 +0000 Subject: [PATCH 382/409] [mockito] clean up false mockito deps Problem go/mockito-problem Solution cleab up some unused deps Differential Revision: https://phabricator.twitter.biz/D967035 --- server/src/test/java/BUILD | 2 -- slf4j-jdk14/src/test/scala/BUILD | 2 -- 2 files changed, 4 deletions(-) diff --git a/server/src/test/java/BUILD b/server/src/test/java/BUILD index 27a0b809..b498fadf 100644 --- a/server/src/test/java/BUILD +++ b/server/src/test/java/BUILD @@ -5,11 +5,9 @@ java_library( compiler_option_sets = [], dependencies = [ "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scala-lang:scala-library", "3rdparty/jvm/org/scalacheck", "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:mockito-1-10", "3rdparty/jvm/org/slf4j:slf4j-api", scoped( "3rdparty/jvm/org/slf4j:slf4j-nop", diff --git a/slf4j-jdk14/src/test/scala/BUILD b/slf4j-jdk14/src/test/scala/BUILD index ecb6d2f1..9ffe1511 100644 --- a/slf4j-jdk14/src/test/scala/BUILD +++ b/slf4j-jdk14/src/test/scala/BUILD @@ -5,11 +5,9 @@ junit_tests( tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scalacheck", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:mockito-1-10", scoped( "twitter-server/slf4j-jdk14/src/main/scala", scope = "runtime", From 0c9bae8b8a2fbde713561b160d6260a1847e9e07 Mon Sep 17 00:00:00 2001 From: Yi Cheng Date: Wed, 7 Sep 2022 20:51:15 +0000 Subject: [PATCH 383/409] [1/N] Migrate top level java_library targets to BUILD.bazel ### Problem / Solution 1. If a library is at the top of the build graph from Pants' perspective, then all its dependees have already migrated. 2. If the BUILD files only contain a library, then it would have no side effects migrating other targets in the same BUILD file. JIRA Issues: DPB-12362 Differential Revision: https://phabricator.twitter.biz/D972539 --- server/src/test/java/{BUILD => BUILD.bazel} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename server/src/test/java/{BUILD => BUILD.bazel} (100%) diff --git a/server/src/test/java/BUILD b/server/src/test/java/BUILD.bazel similarity index 100% rename from server/src/test/java/BUILD rename to server/src/test/java/BUILD.bazel From 23a54a3c77457baac37820521cbfdc701f592e6f Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Fri, 9 Sep 2022 17:36:41 +0000 Subject: [PATCH 384/409] finagle-core: Clean up unused pending value in LB metadata Problem / Solution LB node's "pending" and "load" seems to mean the same thing in our LB stack. The former was never exported as a stat so it only existed as an internal variable. Let's get rid of "pending". JIRA Issues: CSL-12007 Differential Revision: https://phabricator.twitter.biz/D973410 --- .../main/scala/com/twitter/server/view/BalancerHtmlView.scala | 1 - .../main/scala/com/twitter/server/view/BalancersJsonView.scala | 2 -- .../scala/com/twitter/server/view/BalancersJsonViewTest.scala | 1 - 3 files changed, 4 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/view/BalancerHtmlView.scala b/server/src/main/scala/com/twitter/server/view/BalancerHtmlView.scala index a5ac474f..a898bbab 100644 --- a/server/src/main/scala/com/twitter/server/view/BalancerHtmlView.scala +++ b/server/src/main/scala/com/twitter/server/view/BalancerHtmlView.scala @@ -17,7 +17,6 @@ private[server] class BalancerHtmlView(balancer: Option[Metadata], routePath: St | Number Nodes Available${md.numAvailable} | Number Nodes Busy${md.numBusy} | Number Nodes Closed${md.numClosed} - | Total pending requests${md.totalPending} | Total load${md.totalLoad} | Size${md.size} | Panic Mode${md.panicMode} diff --git a/server/src/main/scala/com/twitter/server/view/BalancersJsonView.scala b/server/src/main/scala/com/twitter/server/view/BalancersJsonView.scala index d58fb107..6fdc94c5 100644 --- a/server/src/main/scala/com/twitter/server/view/BalancersJsonView.scala +++ b/server/src/main/scala/com/twitter/server/view/BalancersJsonView.scala @@ -12,7 +12,6 @@ private object BalancersJsonView { numberAvailable: Int, numberBusy: Int, numberClosed: Int, - totalPending: Int, totalLoad: Double, size: Int, panicMode: String, @@ -27,7 +26,6 @@ private object BalancersJsonView { numberAvailable = metadata.numAvailable, numberBusy = metadata.numBusy, numberClosed = metadata.numClosed, - totalPending = metadata.totalPending, totalLoad = metadata.totalLoad, size = metadata.size, panicMode = metadata.panicMode, diff --git a/server/src/test/scala/com/twitter/server/view/BalancersJsonViewTest.scala b/server/src/test/scala/com/twitter/server/view/BalancersJsonViewTest.scala index 4c25d379..7cc97ac3 100644 --- a/server/src/test/scala/com/twitter/server/view/BalancersJsonViewTest.scala +++ b/server/src/test/scala/com/twitter/server/view/BalancersJsonViewTest.scala @@ -37,7 +37,6 @@ class BalancersJsonViewTest extends AnyFunSuite { | "number_available" : 1, | "number_busy" : 0, | "number_closed" : 0, - | "total_pending" : 0, | "total_load" : 0.0, | "size" : 1, | "panic_mode" : "FiftyPercentUnhealthy", From c5a6d6e662615feb202f05c8ebe348c03a22d19a Mon Sep 17 00:00:00 2001 From: Jared Crawford Date: Sat, 24 Sep 2022 00:18:09 +0000 Subject: [PATCH 385/409] [finagle-core] Extract ServiceIdentity from underlying SSLSession Problem Customers often want to define behavior based on an authenticated peer. For example, we may want access controls or rate limits. Solution Extract an X509 based service identity from the local and peer certs when present. Result Customers can get the ServiceIdentity from the underlying UsingSslSessionInfo and build the desired behavior on top of it. JIRA Issues: PSEC-8763 Differential Revision: https://phabricator.twitter.biz/D973030 --- .../handler/AttachedClientsHandlerTest.scala | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/server/src/test/scala/com/twitter/server/handler/AttachedClientsHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/AttachedClientsHandlerTest.scala index 7b405b7a..41303c47 100644 --- a/server/src/test/scala/com/twitter/server/handler/AttachedClientsHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/AttachedClientsHandlerTest.scala @@ -1,14 +1,23 @@ package com.twitter.server.handler import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.http.{Request, Status} +import com.twitter.finagle.http.Request +import com.twitter.finagle.http.Status import com.twitter.finagle.server.ServerRegistry import com.twitter.finagle.service.StatsFilter import com.twitter.finagle.ssl.session.SslSessionInfo -import com.twitter.finagle.{ClientConnection, Service, ServiceFactory, Stack, param} +import com.twitter.finagle.ssl.session.ServiceIdentity +import com.twitter.finagle.ClientConnection +import com.twitter.finagle.Service +import com.twitter.finagle.ServiceFactory +import com.twitter.finagle.Stack +import com.twitter.finagle.param import com.twitter.io.Buf -import com.twitter.util.{Await, Future, Time} -import java.net.{InetSocketAddress, SocketAddress} +import com.twitter.util.Await +import com.twitter.util.Future +import com.twitter.util.Time +import java.net.InetSocketAddress +import java.net.SocketAddress import java.security.Principal import java.security.cert.X509Certificate import java.util.Date @@ -54,6 +63,8 @@ object AttachedClientsHandlerTest { override def cipherSuite: String = "cipher?sweeeeet!" override def localCertificates: Seq[X509Certificate] = ??? override def peerCertificates: Seq[X509Certificate] = Seq(peerCertificate) + override def getLocalIdentity: Option[ServiceIdentity] = None + override def getPeerIdentity: Option[ServiceIdentity] = None } }) } From ab9e3956bf3ed8b97f934225f241b8d0a06ae675 Mon Sep 17 00:00:00 2001 From: Eitan Adler Date: Tue, 27 Sep 2022 17:17:46 +0000 Subject: [PATCH 386/409] [mockito] twitter-server: upgrade mockito deps for twitter-server/server/src/test/scala Problem go/mockito-problem Solution Upgrade mockito to latest JIRA Issues: DEAD-3 Differential Revision: https://phabricator.twitter.biz/D982927 --- server/src/test/scala/BUILD | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/test/scala/BUILD b/server/src/test/scala/BUILD index 7bf616f2..42b041c9 100644 --- a/server/src/test/scala/BUILD +++ b/server/src/test/scala/BUILD @@ -7,11 +7,11 @@ junit_tests( "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", "3rdparty/jvm/com/fasterxml/jackson/module:jackson-module-scala", "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/scalacheck", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:mockito-1-10", + "3rdparty/jvm/org/scalatestplus:mockito", "3rdparty/jvm/org/scalatestplus:scalacheck", "3rdparty/jvm/org/slf4j:slf4j-api", scoped( From 5b39f9cbcc36073cd8f65123a997dcfae77d4328 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Tue, 27 Sep 2022 21:16:52 +0000 Subject: [PATCH 387/409] util-stats: Add notion of an undetermined identity type Problem We want to be able to exclude some metrics from being dimensional because they will become redundant. However, we don't have a way to express that right now. Solution Add an `NonDeterminate` state for our metric identity. This becomes the default state. As metric names this flag can be refined to express that a metric name is only valid hierarchically. JIRA Issues: CSL-12091 Differential Revision: https://phabricator.twitter.biz/D964827 --- .../com/twitter/server/util/MetricSchemaSerializer.scala | 8 +++++++- .../server/handler/MetricMetadataQueryHandlerTest.scala | 5 +++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala b/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala index 1d6fcd7c..eb68cf7c 100644 --- a/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala +++ b/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala @@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.SerializerProvider import com.fasterxml.jackson.databind.ser.std.StdSerializer import com.twitter.finagle.stats.MetricBuilder.HistogramType import com.twitter.finagle.stats.MetricBuilder +import com.twitter.finagle.stats.MetricBuilder.IdentityType import com.twitter.finagle.stats.StatsFormatter import com.twitter.finagle.stats.metadataScopeSeparator @@ -68,7 +69,9 @@ object SchemaSerializer extends StdSerializer[MetricBuilder](classOf[MetricBuild writeDictionary(jsonGenerator, "labels", metricBuilder.identity.labels) - jsonGenerator.writeBooleanField("dimensional_support", !metricBuilder.identity.hierarchicalOnly) + jsonGenerator.writeBooleanField( + "dimensional_support", + dimensionalSupport(metricBuilder.identity.identityType)) val dataType = metricBuilder.metricType.toJsonString jsonGenerator.writeStringField("kind", dataType) @@ -110,6 +113,9 @@ object SchemaSerializer extends StdSerializer[MetricBuilder](classOf[MetricBuild jsonGenerator.writeEndObject() } + private[this] def dimensionalSupport(identityType: IdentityType): Boolean = + IdentityType.toResolvedIdentityType(identityType) == IdentityType.Full + private def convertNullToString(segment: String): String = { if (segment == null) "null" else segment diff --git a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala index 7f4d8b91..fee0f39e 100644 --- a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala @@ -5,6 +5,7 @@ import com.twitter.finagle.stats.MetricBuilder.CounterType import com.twitter.finagle.stats.MetricBuilder.GaugeType import com.twitter.finagle.stats.MetricBuilder.HistogramType import com.twitter.finagle.stats.MetricBuilder.Identity +import com.twitter.finagle.stats.MetricBuilder.IdentityType import com.twitter.finagle.stats._ import com.twitter.finagle.stats.exp.ExpressionSchema import com.twitter.finagle.stats.exp.ExpressionSchemaKey @@ -34,7 +35,7 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { Seq("my", "cool", "counter"), Seq("my", "cool", "counter"), Map("label1" -> "value1", "label2" -> "value2"), - hierarchicalOnly = false + identityType = IdentityType.Full )), "your/fine/gauge" -> MetricBuilder( @@ -59,7 +60,7 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite { Seq("my", "only", "histo"), Seq("my", "only", "histo"), Map("label1" -> "value1"), - hierarchicalOnly = false, + identityType = IdentityType.Full )), "my/bad/null/counter" -> MetricBuilder( From cb8799e08a1ea7fdc4ea5c9203d7e5a7b4330a3d Mon Sep 17 00:00:00 2001 From: Ruben Oanta Date: Wed, 5 Oct 2022 23:52:57 +0000 Subject: [PATCH 388/409] twitter-server: apply IndexView to all combined muxers Problem We only apply IndexView to routes added statically via the `Admin` trait. This means that any routes which are added via the global HttpMuxer can't take advantage of the styles and templating inside of `IndexView`. Solution Move combine inside of `AdminHttpServer` and apply the filter dynamically inside of it. Result `IndexView` is now applied to all the muxers combined in the `AdminHttpServer`. Differential Revision: https://phabricator.twitter.biz/D989672 --- .../com/twitter/server/AdminHttpServer.scala | 48 ++++-- .../com/twitter/server/util/HttpUtils.scala | 27 +--- .../com/twitter/server/view/IndexView.scala | 37 +++-- .../twitter/server/AdminHttpServerTest.scala | 144 +++++++++++++++++- .../twitter/server/util/HttpUtilsTest.scala | 138 +---------------- 5 files changed, 214 insertions(+), 180 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/AdminHttpServer.scala b/server/src/main/scala/com/twitter/server/AdminHttpServer.scala index 0b9f69a3..cdf55fce 100644 --- a/server/src/main/scala/com/twitter/server/AdminHttpServer.scala +++ b/server/src/main/scala/com/twitter/server/AdminHttpServer.scala @@ -8,6 +8,7 @@ import com.twitter.finagle.http.HttpMuxer import com.twitter.finagle.http.Method import com.twitter.finagle.http.Request import com.twitter.finagle.http.Response +import com.twitter.finagle.http.Status import com.twitter.finagle.http.{Route => HttpRoute} import com.twitter.finagle.server.ServerRegistry import com.twitter.finagle.stats.NullStatsReceiver @@ -32,6 +33,7 @@ import com.twitter.util.registry.Library import com.twitter.util.Future import com.twitter.util.Monitor import com.twitter.util.Time +import com.twitter.util.Closable import java.net.InetSocketAddress import java.net.URLEncoder import java.nio.charset.StandardCharsets @@ -131,6 +133,38 @@ object AdminHttpServer { includeInIndex = true ) ) + + /** + * Creates a Finagle [[Service]] which dispatches requests across the given `muxers` based + * on path/pattern matching. This is used internally inside of `updateMuxers` and its semantics + * are coupled to the admin http server. For example, we apply an `IndexView` to each request + * which is dispatched. + * + * @note This has historically been done dynamically (i.e. per-dispatch) instead of snapshotting + * the contents of the muxers. We are dependent on this behavior in various places throughout + * are code. + */ + private[server] def combine( + muxers: Seq[HttpMuxer], + indexEntries: () => Seq[IndexView.Entry] + ): Service[Request, Response] = + new Service[Request, Response] { + def apply(req: Request): Future[Response] = { + val routes = muxers.flatMap(_.route(req)) + if (routes.isEmpty) { + Future.value(Response(req.version, Status.NotFound)) + } else { + val route = routes.maxBy(_.pattern.length) + val alias = route.index.map(_.alias).getOrElse(route.pattern) + val indexFilter = new IndexView(alias, route.pattern, indexEntries) + val svc = indexFilter.andThen(route.handler) + svc(req) + } + } + + override def close(deadline: Time): Future[Unit] = + Closable.all(muxers: _*).close(deadline) + } } /** @@ -255,18 +289,14 @@ trait AdminHttpServer { self: App with Stats => protected def configureAdminHttpServer(server: Http.Server): Http.Server = server private[this] def updateMuxer(): Unit = { - // create a service which multiplexes across all endpoints. + val endpoints = allRoutes.map { route => s"\t${route.path} => ${route.handler.toString}" } + log.debug(s"AdminHttpServer Muxer endpoints:\n" + endpoints.mkString("\n")) + val localMuxer = allRoutes.foldLeft(new HttpMuxer) { - case (muxer, route) => - val service = - new IndexView(route.alias, route.path, () => indexEntries).andThen(route.handler) - muxer.withHandler(route.path, service) + case (muxer, route) => muxer.withHandler(route.path, route.handler) } - val endpoints = allRoutes.map { route => s"\t${route.path} => ${route.handler.toString}" } - - log.debug(s"AdminHttpServer Muxer endpoints:\n" + endpoints.mkString("\n")) - adminHttpMuxer.underlying = HttpUtils.combine(Seq(localMuxer, HttpMuxer)) + adminHttpMuxer.underlying = combine(Seq(localMuxer, HttpMuxer), () => indexEntries) } /** create index with both the local and global muxer namespace and server/client registries. */ diff --git a/server/src/main/scala/com/twitter/server/util/HttpUtils.scala b/server/src/main/scala/com/twitter/server/util/HttpUtils.scala index ed8cf7e5..9b372982 100644 --- a/server/src/main/scala/com/twitter/server/util/HttpUtils.scala +++ b/server/src/main/scala/com/twitter/server/util/HttpUtils.scala @@ -1,31 +1,16 @@ package com.twitter.server.util -import com.twitter.finagle.Service -import com.twitter.finagle.http.{MediaType, Request, Response, Status, Version, HttpMuxer} +import com.twitter.finagle.http.MediaType +import com.twitter.finagle.http.Request +import com.twitter.finagle.http.Response +import com.twitter.finagle.http.Status +import com.twitter.finagle.http.Version import com.twitter.io.Buf -import com.twitter.util.{Future, Time, Closable} +import com.twitter.util.Future import io.netty.handler.codec.http.QueryStringDecoder private[server] object HttpUtils { - /** - * Creates a http [[com.twitter.finagle.Service]] which attempts a - * request on the given `services`, in order, until a service returns - * a response with a non-404 status code. If none return a non-404, - * the response of the last service is used. If the given list of `services` - * is empty the resulting services will be always answering with 404. - */ - def combine(services: Seq[HttpMuxer]): Service[Request, Response] = - new Service[Request, Response] { - def apply(req: Request): Future[Response] = { - val routes = services.flatMap(_.route(req)) - if (routes.nonEmpty) routes.maxBy(_.pattern.length).handler(req) - else Future.value(Response(req.version, Status.NotFound)) - } - - override def close(deadline: Time): Future[Unit] = Closable.all(services: _*).close(deadline) - } - /** * Determines (by examine the "Accept" header on `req`) if the client accepts given `contentType`. * diff --git a/server/src/main/scala/com/twitter/server/view/IndexView.scala b/server/src/main/scala/com/twitter/server/view/IndexView.scala index 5502b26b..1f7caaf8 100644 --- a/server/src/main/scala/com/twitter/server/view/IndexView.scala +++ b/server/src/main/scala/com/twitter/server/view/IndexView.scala @@ -2,12 +2,19 @@ package com.twitter.server.view import com.twitter.concurrent.AsyncStream import com.twitter.finagle.http.Method -import com.twitter.finagle.http.Method.{Get, Post} -import com.twitter.finagle.{Service, SimpleFilter} -import com.twitter.finagle.http.{Request, Response} -import com.twitter.io.{Buf, BufReader, Pipe, Reader} +import com.twitter.finagle.http.Method.Get +import com.twitter.finagle.http.Method.Post +import com.twitter.finagle.Service +import com.twitter.finagle.SimpleFilter +import com.twitter.finagle.http.Request +import com.twitter.finagle.http.Response +import com.twitter.io.Buf +import com.twitter.io.BufReader +import com.twitter.io.Pipe +import com.twitter.io.Reader import com.twitter.server.util.HtmlUtils.escapeHtml -import com.twitter.server.util.HttpUtils.{expectsHtml, newResponse} +import com.twitter.server.util.HttpUtils.expectsHtml +import com.twitter.server.util.HttpUtils.newResponse import com.twitter.util.Future object IndexView { @@ -84,7 +91,7 @@ object IndexView { Reader.concat( Reader.fromBuf( Buf.Utf8( - s""" + s""" ${title} · Twitter Server Admin @@ -119,6 +126,18 @@ object IndexView { """))) ) } + + /** + * Returns "true" if the contents of the http response is determined to be a fragment (i.e. + * it can/should be nested in an IndexView). This is currently based on the "content-type" + * header and some heuristics on the `content`. We should consider making this explicit and + * set a header on the response from the twitter-server handlers. + */ + private def isFragment(contentType: String, content: String): Boolean = { + contentType.toLowerCase.contains("text/html") && + !content.contains("") && + !content.contains("") + } } /** @@ -131,10 +150,6 @@ class IndexView(title: String, uri: String, index: () => Seq[IndexView.Entry]) extends SimpleFilter[Request, Response] { import IndexView._ - private[this] def isFragment(contentType: String, content: String): Boolean = { - contentType.toLowerCase.contains("text/html") && !content.contains("") - } - def apply(req: Request, svc: Service[Request, Response]): Future[Response] = if (!expectsHtml(req)) svc(req) else @@ -151,7 +166,7 @@ class IndexView(title: String, uri: String, index: () => Seq[IndexView.Entry]) Pipe.copy(reader, response.writer) ensure response.writer.close() Future.value(response) - case res => + case _ => val body = Reader.fromBuf(Buf.Utf8(content)) val reader = render(title, uri, index().sorted, body) BufReader.readAll(reader).flatMap { html => diff --git a/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala b/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala index c59f6e8b..4bfc2c9b 100644 --- a/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala +++ b/server/src/test/scala/com/twitter/server/AdminHttpServerTest.scala @@ -2,6 +2,7 @@ package com.twitter.server import com.twitter.conversions.DurationOps._ import com.twitter.finagle.Http +import com.twitter.finagle.Service import com.twitter.finagle.ListeningServer import com.twitter.finagle.http._ import com.twitter.server.util.HttpUtils._ @@ -58,25 +59,26 @@ class MockClosableHandler extends HttpMuxHandler { } class AdminHttpServerTest extends AnyFunSuite with Eventually with IntegrationPatience { + private[this] def await[A](a: Future[A]): A = Await.result(a, 5.seconds) def checkServer(server: ListeningServer, shadow: Boolean = false): Unit = { val port = server.boundAddress.asInstanceOf[InetSocketAddress].getPort val client = Http.client.newService(s"localhost:$port") - val ostrich = Await.result(client(Request("/stats.json")), 1.second) + val ostrich = await(client(Request("/stats.json"))) assert(ostrich.contentString.contains("metrics!")) - val commons = Await.result(client(Request("/vars.json")), 1.second) + val commons = await(client(Request("/vars.json"))) assert(commons.contentString.contains("commons stats!")) - val resp = Await.result(client(Request("/admin/metrics.json")), 1.second) + val resp = await(client(Request("/admin/metrics.json"))) assert(resp.contentString.contains("standard metrics!")) - val resp1 = Await.result(client(Request("/admin/per_host_metrics.json")), 1.second) + val resp1 = await(client(Request("/admin/per_host_metrics.json"))) assert(resp1.contentString.contains("per host metrics!")) if (!shadow) { - val resp2 = Await.result(client(Request("/admin/closable.json")), 1.second) + val resp2 = await(client(Request("/admin/closable.json"))) assert(resp2.contentString.contains("closable!")) } } @@ -85,7 +87,7 @@ class AdminHttpServerTest extends AnyFunSuite with Eventually with IntegrationPa val adminServerBoundPort = adminServer.boundAddress.asInstanceOf[InetSocketAddress].getPort assert(adminServerBoundPort == twitterServer.adminBoundAddress.getPort) val client = Http.client.newService(s"localhost:$adminServerBoundPort") - Await.result(client(Request(Method.Post, "/quitquitquit")), 1.second) + await(client(Request(Method.Post, "/quitquitquit"))) // throws if adminHttpServer does not exit before the grace period, // which indicates that we have not closed it properly. @@ -254,4 +256,134 @@ class AdminHttpServerTest extends AnyFunSuite with Eventually with IntegrationPa server.main(args = Array.empty[String]) assert(server.isAdminServer) } + + test("combine can combine two muxers") { + val hello = new Service[Request, Response] { + def apply(req: Request) = newOk("hello") + } + + val world = new Service[Request, Response] { + def apply(req: Request) = newOk("world") + } + + val muxer0 = new HttpMuxer().withHandler("/hello", hello) + val muxer1 = new HttpMuxer().withHandler("/world", world) + + val svc = AdminHttpServer.combine(Seq(muxer0, muxer1), () => Seq.empty) + + val res0 = await(svc(Request("/hello"))) + assert(res0.contentString == "hello") + + val res1 = await(svc(Request("/world"))) + assert(res1.contentString == "world") + } + + test("combine order doesn't matter for different paths") { + val hello = new Service[Request, Response] { + def apply(req: Request) = newOk("hello") + } + + val world = new Service[Request, Response] { + def apply(req: Request) = newOk("world") + } + + val muxer0 = new HttpMuxer().withHandler("/hello", hello) + val muxer1 = new HttpMuxer().withHandler("/world", world) + + Seq(muxer0, muxer1).permutations.foreach { seq => + val svc = AdminHttpServer.combine(seq, () => Seq.empty) + + val res0 = await(svc(Request("/hello"))) + assert(res0.contentString == "hello") + + val res1 = await(svc(Request("/world"))) + assert(res1.contentString == "world") + } + } + + test("combine always chooses the longer prefix") { + val hello = new Service[Request, Response] { + def apply(req: Request) = newOk("hello") + } + + val world = new Service[Request, Response] { + def apply(req: Request) = newOk("world") + } + + val muxer0 = new HttpMuxer().withHandler("/hello", hello) + val muxer1 = new HttpMuxer().withHandler("/hello1", world) + + Seq(muxer0, muxer1).permutations.foreach { seq => + val svc = AdminHttpServer.combine(seq, () => Seq.empty) + + val res = await(svc(Request("/hello1"))) + assert(res.contentString == "world") + } + } + + test("combine order is respected for identical paths") { + val hello = new Service[Request, Response] { + def apply(req: Request) = newOk("hello") + } + + val world = new Service[Request, Response] { + def apply(req: Request) = newOk("world") + } + val muxer0 = new HttpMuxer().withHandler("/hello", hello) + val muxer1 = new HttpMuxer().withHandler("/hello", world) + + val svcSeq0 = AdminHttpServer.combine(Seq(muxer0, muxer1), () => Seq.empty) + val res0 = await(svcSeq0(Request("/hello"))) + assert(res0.contentString == "hello") + + val svcSeq1 = AdminHttpServer.combine(Seq(muxer1, muxer0), () => Seq.empty) + val res1 = await(svcSeq1(Request("/hello"))) + assert(res1.contentString == "world") + } + + test("combine can 404 properly") { + val hello = new Service[Request, Response] { + def apply(req: Request) = newOk("hello") + } + + val world = new Service[Request, Response] { + def apply(req: Request) = newOk("world") + } + val muxer0 = new HttpMuxer().withHandler("/hello", hello) + val muxer1 = new HttpMuxer().withHandler("/world", world) + + val svc = AdminHttpServer.combine(Seq(muxer0, muxer1), () => Seq.empty) + + val res = await(svc(Request("/an404"))) + assert(res.status == Status.NotFound) + } + + test("combine can close the underlying services") { + var closed1 = false + var closed2 = false + + val hello = new Service[Request, Response] { + def apply(req: Request) = newOk("hello") + override def close(deadline: Time): Future[Unit] = { + closed1 = true + Future.Done + } + } + + val world = new Service[Request, Response] { + def apply(req: Request) = newOk("world") + override def close(deadline: Time): Future[Unit] = { + closed2 = true + Future.Done + } + } + val muxer0 = new HttpMuxer().withHandler("/hello", hello) + val muxer1 = new HttpMuxer().withHandler("/world", world) + + val svc = AdminHttpServer.combine(Seq(muxer0, muxer1), () => Seq.empty) + await(svc.close()) + + assert(closed1) + assert(closed2) + } } diff --git a/server/src/test/scala/com/twitter/server/util/HttpUtilsTest.scala b/server/src/test/scala/com/twitter/server/util/HttpUtilsTest.scala index c6d8deb4..235d1f75 100644 --- a/server/src/test/scala/com/twitter/server/util/HttpUtilsTest.scala +++ b/server/src/test/scala/com/twitter/server/util/HttpUtilsTest.scala @@ -1,146 +1,18 @@ package com.twitter.server.util import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.Service -import com.twitter.finagle.http.{HttpMuxer, Request, Response, Status, Version} +import com.twitter.finagle.http.Request +import com.twitter.finagle.http.Status +import com.twitter.finagle.http.Version import com.twitter.io.Buf import com.twitter.server.util.HttpUtils._ -import com.twitter.util.{Await, Future, Time} +import com.twitter.util.Await +import com.twitter.util.Future import org.scalatest.funsuite.AnyFunSuite class HttpUtilsTest extends AnyFunSuite { private[this] def await[A](a: Future[A]): A = Await.result(a, 5.seconds) - test("combine can combine two muxers") { - val hello = new Service[Request, Response] { - def apply(req: Request) = newOk("hello") - } - - val world = new Service[Request, Response] { - def apply(req: Request) = newOk("world") - } - - val muxer0 = new HttpMuxer().withHandler("/hello", hello) - val muxer1 = new HttpMuxer().withHandler("/world", world) - - val svc = combine(Seq(muxer0, muxer1)) - - val res0 = await(svc(Request("/hello"))) - assert(res0.contentString == "hello") - - val res1 = await(svc(Request("/world"))) - assert(res1.contentString == "world") - } - - test("combine order doesn't matter for different paths") { - val hello = new Service[Request, Response] { - def apply(req: Request) = newOk("hello") - } - - val world = new Service[Request, Response] { - def apply(req: Request) = newOk("world") - } - - val muxer0 = new HttpMuxer().withHandler("/hello", hello) - val muxer1 = new HttpMuxer().withHandler("/world", world) - - Seq(muxer0, muxer1).permutations.foreach { seq => - val svc = combine(seq) - - val res0 = await(svc(Request("/hello"))) - assert(res0.contentString == "hello") - - val res1 = await(svc(Request("/world"))) - assert(res1.contentString == "world") - } - } - - test("combine always chooses the longer prefix") { - val hello = new Service[Request, Response] { - def apply(req: Request) = newOk("hello") - } - - val world = new Service[Request, Response] { - def apply(req: Request) = newOk("world") - } - - val muxer0 = new HttpMuxer().withHandler("/hello", hello) - val muxer1 = new HttpMuxer().withHandler("/hello1", world) - - Seq(muxer0, muxer1).permutations.foreach { seq => - val svc = combine(seq) - - val res = await(svc(Request("/hello1"))) - assert(res.contentString == "world") - } - } - - test("combine order is respected for identical paths") { - val hello = new Service[Request, Response] { - def apply(req: Request) = newOk("hello") - } - - val world = new Service[Request, Response] { - def apply(req: Request) = newOk("world") - } - val muxer0 = new HttpMuxer().withHandler("/hello", hello) - val muxer1 = new HttpMuxer().withHandler("/hello", world) - - val svcSeq0 = combine(Seq(muxer0, muxer1)) - val res0 = await(svcSeq0(Request("/hello"))) - assert(res0.contentString == "hello") - - val svcSeq1 = combine(Seq(muxer1, muxer0)) - val res1 = await(svcSeq1(Request("/hello"))) - assert(res1.contentString == "world") - } - - test("combine can 404 properly") { - val hello = new Service[Request, Response] { - def apply(req: Request) = newOk("hello") - } - - val world = new Service[Request, Response] { - def apply(req: Request) = newOk("world") - } - val muxer0 = new HttpMuxer().withHandler("/hello", hello) - val muxer1 = new HttpMuxer().withHandler("/world", world) - - val svc = combine(Seq(muxer0, muxer1)) - - val res = await(svc(Request("/an404"))) - assert(res.status == Status.NotFound) - } - - test("combine can close the underlying services") { - var closed1 = false - var closed2 = false - - val hello = new Service[Request, Response] { - def apply(req: Request) = newOk("hello") - override def close(deadline: Time): Future[Unit] = { - closed1 = true - Future.Done - } - } - - val world = new Service[Request, Response] { - def apply(req: Request) = newOk("world") - override def close(deadline: Time): Future[Unit] = { - closed2 = true - Future.Done - } - } - val muxer0 = new HttpMuxer().withHandler("/hello", hello) - val muxer1 = new HttpMuxer().withHandler("/world", world) - - val svc = combine(Seq(muxer0, muxer1)) - await(svc.close()) - - assert(closed1) - assert(closed2) - } - test("expects") { val req1 = Request("/") req1.headerMap From 08940296cec04795766c7b53ab34d541b099152f Mon Sep 17 00:00:00 2001 From: Yi Cheng Date: Fri, 7 Oct 2022 19:22:56 +0000 Subject: [PATCH 389/409] [bazel] move more junit_tests to bazel only Differential Revision: https://phabricator.twitter.biz/D991456 --- server/src/test/scala/{BUILD => BUILD.bazel} | 0 slf4j-jdk14/src/test/scala/{BUILD => BUILD.bazel} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename server/src/test/scala/{BUILD => BUILD.bazel} (100%) rename slf4j-jdk14/src/test/scala/{BUILD => BUILD.bazel} (100%) diff --git a/server/src/test/scala/BUILD b/server/src/test/scala/BUILD.bazel similarity index 100% rename from server/src/test/scala/BUILD rename to server/src/test/scala/BUILD.bazel diff --git a/slf4j-jdk14/src/test/scala/BUILD b/slf4j-jdk14/src/test/scala/BUILD.bazel similarity index 100% rename from slf4j-jdk14/src/test/scala/BUILD rename to slf4j-jdk14/src/test/scala/BUILD.bazel From 675ff365146da8cb05b9d95a69c4d755ed1e3640 Mon Sep 17 00:00:00 2001 From: Yi Cheng Date: Sun, 16 Oct 2022 23:02:52 +0000 Subject: [PATCH 390/409] [bazel] convert applicable resources target to bazel only Differential Revision: https://phabricator.twitter.biz/D999259 --- server/src/test/resources/{BUILD => BUILD.bazel} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename server/src/test/resources/{BUILD => BUILD.bazel} (100%) diff --git a/server/src/test/resources/BUILD b/server/src/test/resources/BUILD.bazel similarity index 100% rename from server/src/test/resources/BUILD rename to server/src/test/resources/BUILD.bazel From fc2959b67b6fde77fa957162e448c7fa76944325 Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Tue, 18 Oct 2022 23:57:03 +0000 Subject: [PATCH 391/409] twitter-server: Expose namespaces in admin/namespaces endpoint Problem/Solution Expose namespaces carried in `ExpressionSchema` via a new Twitter Server Admin endpoint as `admin/namespaces` JIRA Issues: CSL-11983 Differential Revision: https://phabricator.twitter.biz/D986107 --- .../main/scala/com/twitter/server/Admin.scala | 17 ++-- .../server/handler/NamespaceHandler.scala | 48 ++++++++++ .../server/util/MetricSchemaSource.scala | 6 ++ .../handler/exp/NamespaceHandlerTest.scala | 88 +++++++++++++++++++ 4 files changed, 153 insertions(+), 6 deletions(-) create mode 100644 server/src/main/scala/com/twitter/server/handler/NamespaceHandler.scala create mode 100644 server/src/test/scala/com/twitter/server/handler/exp/NamespaceHandlerTest.scala diff --git a/server/src/main/scala/com/twitter/server/Admin.scala b/server/src/main/scala/com/twitter/server/Admin.scala index 366df7b6..6f384318 100644 --- a/server/src/main/scala/com/twitter/server/Admin.scala +++ b/server/src/main/scala/com/twitter/server/Admin.scala @@ -2,12 +2,10 @@ package com.twitter.server import com.twitter.app.App import com.twitter.finagle.http.Method -import com.twitter.finagle.stats.{ - AggregateWithHistogramDetails, - DelegatingStatsReceiver, - StatsReceiver, - WithHistogramDetails -} +import com.twitter.finagle.stats.AggregateWithHistogramDetails +import com.twitter.finagle.stats.DelegatingStatsReceiver +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.finagle.stats.WithHistogramDetails import com.twitter.server.AdminHttpServer.Route import com.twitter.server.filters.AdminThreadPoolFilter import com.twitter.server.handler._ @@ -252,6 +250,13 @@ object Admin { alias = "Incoming Connections", group = Some(Grouping.Utilities), includeInIndex = true + ), + Route( + path = "admin/namespaces", + handler = new NamespaceHandler(), + alias = "Namespaces in chronological order", + group = None, + includeInIndex = false ) ).map(AdminThreadPoolFilter.isolateRoute) diff --git a/server/src/main/scala/com/twitter/server/handler/NamespaceHandler.scala b/server/src/main/scala/com/twitter/server/handler/NamespaceHandler.scala new file mode 100644 index 00000000..f7b8f73f --- /dev/null +++ b/server/src/main/scala/com/twitter/server/handler/NamespaceHandler.scala @@ -0,0 +1,48 @@ +package com.twitter.server.handler + +import com.twitter.finagle.Service +import com.twitter.finagle.http.MediaType +import com.twitter.finagle.http.Request +import com.twitter.finagle.http.Response +import com.twitter.io.Buf +import com.twitter.server.util.HttpUtils.newResponse +import com.twitter.server.util.AdminJsonConverter +import com.twitter.server.util.MetricSchemaSource +import com.twitter.util.Future + +/** + * A handler which returns all de-duplicated namespaces carried in the + * requests to a Twitter Server instance. + * + * Example Request: + * http://$HOST:$PORT/admin/namespaces + * + * Response: + * { + "@version" : 1.0, + "namespaces" : [ + { "name": "my/namespace/0" }, + { "name": "my/namespace/1" }, + { "name": "my/namespace/2" }, + { "name": "my/namespace/3" } + ]} + */ +class NamespaceHandler(source: MetricSchemaSource = new MetricSchemaSource) + extends Service[Request, Response] { + + def apply(req: Request): Future[Response] = { + newResponse( + contentType = MediaType.JsonUtf8, + content = Buf.Utf8( + AdminJsonConverter.writeToString( + Map( + "@version" -> 1.0, + "namespaces" -> buildNamespace(source.namespaces) + ))) + ) + } + + private[this] def buildNamespace(namespaces: Set[String]) = + source.namespaces.map { namespace => Map("name" -> namespace) } + +} diff --git a/server/src/main/scala/com/twitter/server/util/MetricSchemaSource.scala b/server/src/main/scala/com/twitter/server/util/MetricSchemaSource.scala index d7326fbd..046755b1 100644 --- a/server/src/main/scala/com/twitter/server/util/MetricSchemaSource.scala +++ b/server/src/main/scala/com/twitter/server/util/MetricSchemaSource.scala @@ -63,4 +63,10 @@ private[server] class MetricSchemaSource( } } } + + def namespaces: Set[String] = synchronized { + registry.foldLeft(Set[String]()) { (set, r) => + set ++ r.expressions().keySet.flatMap(_.namespaces) + } + } } diff --git a/server/src/test/scala/com/twitter/server/handler/exp/NamespaceHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/exp/NamespaceHandlerTest.scala new file mode 100644 index 00000000..f30feff2 --- /dev/null +++ b/server/src/test/scala/com/twitter/server/handler/exp/NamespaceHandlerTest.scala @@ -0,0 +1,88 @@ +package com.twitter.server.handler.exp + +import com.twitter.conversions.DurationOps._ +import com.twitter.finagle.http.Request +import com.twitter.finagle.stats.SchemaRegistry +import com.twitter.server.util.JsonUtils +import com.twitter.server.util.MetricSchemaSource +import com.twitter.server.handler.NamespaceHandler +import com.twitter.finagle.stats.MetricBuilder +import com.twitter.finagle.stats.exp.ExpressionSchema +import com.twitter.finagle.stats.exp.ExpressionSchemaKey +import com.twitter.util.Await +import com.twitter.util.Awaitable +import com.twitter.util.Duration +import org.scalatest.funsuite.AnyFunSuite +import com.twitter.finagle.stats.MetricBuilder.CounterType +import com.twitter.finagle.stats.MetricBuilder.HistogramType +import com.twitter.finagle.stats.SchemaRegistry +import com.twitter.finagle.stats.exp.Expression +import com.twitter.finagle.stats.exp.ExpressionSchema +import com.twitter.finagle.stats.exp.ExpressionSchemaKey +import com.twitter.finagle.stats.exp.GreaterThan +import com.twitter.finagle.stats.exp.HistogramComponent +import com.twitter.finagle.stats.exp.MonotoneThresholds +import NamespaceHandlerTest._ + +object NamespaceHandlerTest { + private val successMb = MetricBuilder(name = Seq("success"), metricType = CounterType) + private val failuresMb = + MetricBuilder(name = Seq("failures"), metricType = CounterType) + private val latencyMb = + MetricBuilder(name = Seq("latency"), metricType = HistogramType) + + private val successRateExpression = + ExpressionSchema( + "success_rate", + Expression(100).multiply( + Expression(successMb).divide(Expression(successMb).plus(Expression(failuresMb)))) + ).withBounds(MonotoneThresholds(GreaterThan, 99.5, 99.97)) + + private val throughputExpression = + ExpressionSchema("throughput", Expression(successMb).plus(Expression(failuresMb))) + .withNamespace("my/namespace/0", "my/namespace/1", "my/namespace/2") + + private val latencyP99 = + ExpressionSchema("latency_p99", Expression(latencyMb, HistogramComponent.Percentile(0.99))) + .withNamespace("my/namespace/0", "my/namespace/3") + + private val expressionSchemaMap: Map[ExpressionSchemaKey, ExpressionSchema] = Map( + ExpressionSchemaKey("success_rate", Map(), Seq()) -> successRateExpression, + ExpressionSchemaKey( + "throughput", + Map(), + Seq("my/namespace/0", "my/namespace/1", "my/namespace/2")) -> throughputExpression, + ExpressionSchemaKey("latency", Map(), Seq("my/namespace/0", "my/namespace/3")) -> latencyP99 + ) + val NamespaceRegistry = new SchemaRegistry { + val hasLatchedCounters = false + def schemas(): Map[String, MetricBuilder] = Map.empty + def expressions(): Map[ExpressionSchemaKey, ExpressionSchema] = expressionSchemaMap + } +} + +class NamespaceHandlerTest extends AnyFunSuite { + + val namespaceSource = new MetricSchemaSource(Seq(NamespaceRegistry), sort = true) + val namespaceHandler = new NamespaceHandler(namespaceSource) + val testRequest = Request("http://$HOST:$PORT/admin/namespaces") + + test("Get all namespaces") { + val expectedResponse = + """ + |{ "@version": 1.0, + | "namespaces": [ + | { "name": "my/namespace/0" }, + | { "name": "my/namespace/1" }, + | { "name": "my/namespace/2" }, + | { "name": "my/namespace/3" } + |]}""".stripMargin + + JsonUtils.assertJsonResponse( + await(namespaceHandler(testRequest)).contentString, + expectedResponse) + } + + private[this] def await[T](awaitable: Awaitable[T], timeout: Duration = 5.second): T = + Await.result(awaitable, timeout) +} From 2db9bf6eff8091305d42302ebd8c25b605917fb9 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Fri, 4 Nov 2022 01:26:17 +0000 Subject: [PATCH 392/409] finagle, twitter-server: Fix some tests that are flaky with OffloadFilter enabled Problem Some tests have faulty assumptions about event ordering that breaks down with OffloadFilter enabled. That is preventing us from changing the default behavior. Solution Fix the tests. Differential Revision: https://phabricator.twitter.biz/D1011251 --- .../com/twitter/server/HandlersTest.scala | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/slf4j-jdk14/src/test/scala/com/twitter/server/HandlersTest.scala b/slf4j-jdk14/src/test/scala/com/twitter/server/HandlersTest.scala index 5a82b96d..ed54f531 100644 --- a/slf4j-jdk14/src/test/scala/com/twitter/server/HandlersTest.scala +++ b/slf4j-jdk14/src/test/scala/com/twitter/server/HandlersTest.scala @@ -1,11 +1,17 @@ package com.twitter.server -import com.twitter.finagle.http.{Request, Response} -import com.twitter.finagle.{Http, Service} -import com.twitter.util.{Await, Future} +import com.twitter.finagle.filter.OffloadFilter +import com.twitter.finagle.http.Request +import com.twitter.finagle.http.Response +import com.twitter.finagle.Http +import com.twitter.finagle.Service +import com.twitter.util.Await +import com.twitter.util.Future import java.io.ByteArrayOutputStream import java.net.InetSocketAddress -import java.util.logging.{Logger, SimpleFormatter, StreamHandler} +import java.util.logging.Logger +import java.util.logging.SimpleFormatter +import java.util.logging.StreamHandler import scala.collection.mutable import org.scalatest.funsuite.AnyFunSuite @@ -49,6 +55,11 @@ class HandlersTest extends AnyFunSuite { val twitterServer: TwitterServer = new TestTwitterServer { val mockExceptionHandler = new MockExceptionHandler + override protected def configureAdminHttpServer(server: Http.Server): Http.Server = { + // TODO: with offload filter there appears to be a race condition in the test. + super.configureAdminHttpServer(server).configured(OffloadFilter.Param.Disabled) + } + override def main(): Unit = { addAdminRoute( AdminHttpServer.mkRoute( From 39a6d0f0ae9b8b154a8b3d18a6792ea4b32bf084 Mon Sep 17 00:00:00 2001 From: Jillian Crossley Date: Thu, 24 Nov 2022 19:15:24 +0000 Subject: [PATCH 393/409] Clean up CSL-owned PROJECT files Problem The owners in these files are no longer maintainers. Solution Clean up files and add new owners. Differential Revision: https://phabricator.twitter.biz/D1016550 --- PROJECT.yml | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/PROJECT.yml b/PROJECT.yml index 00453376..c9d15276 100644 --- a/PROJECT.yml +++ b/PROJECT.yml @@ -1,17 +1,7 @@ # See go/project-docs. Generated using ./pants run src/python/twitter/phabricator:owners_to_project -- --dirs=a/owned/path,b/owned/path owners: - - banderson - - hallam - - ibennett - - jbestourous - - jyan - - mnakamura - - roanta - - yufang -watchers: - csl-team:ldap - -fileOwners: - CHANGELOG.rst: - - csl-team:ldap + - jillianc + - mbezoyan + - nvong From 3f10fbe8dffeebdbb324088b09356126b1530128 Mon Sep 17 00:00:00 2001 From: Matthew Dickinson Date: Tue, 20 Dec 2022 00:11:47 +0000 Subject: [PATCH 394/409] [logback] bump version to 1.2.11 JIRA Issues: DSTR-11387 Differential Revision: https://phabricator.twitter.biz/D1026765 --- CHANGELOG.rst | 6 ++++++ build.sbt | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ca596abc..b9cafbd5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,12 @@ Unreleased * During warmup, the /health route will respond with a more informative "warming up" message, previously an empty message was used. ``PHAB_ID=D949333`` +Runtime Behavior Changes +~~~~~~~~~~~~~~~~~~~~~~~~ + +* util: Bump version of logback to 1.2.11. ``PHAB_ID=D1026765`` + + 22.7.0 ------ diff --git a/build.sbt b/build.sbt index f8502e69..9e875428 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch val releaseVersion = "22.10.0-SNAPSHOT" -val logbackVersion = "1.2.8" +val logbackVersion = "1.2.11" val opencensusVersion = "0.19.1" val slf4jVersion = "1.7.30" From f0e6baaee99c6e300b7175764df6660ce5fec504 Mon Sep 17 00:00:00 2001 From: Matthew Dickinson Date: Fri, 23 Dec 2022 17:25:13 +0000 Subject: [PATCH 395/409] Twitter-oss: Prepare OSS libraries for release 22.12.0 Problem We want to release the next versions of our Twitter OSS libraries 22.12.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D1026958 --- CHANGELOG.rst | 3 +++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b9cafbd5..5cb3cac7 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +22.12.0 +------- + * During warmup, the /health route will respond with a more informative "warming up" message, previously an empty message was used. ``PHAB_ID=D949333`` diff --git a/README.md b/README.md index 39a9998d..50505902 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) [![Maven Central](https://maven-badges.herokuapp.com/maven-central/com.twitter/twitter-server_2.12/badge.svg)](https://maven-badges.herokuapp.com/maven-central/com.twitter/twitter-server_2.12) diff --git a/build.sbt b/build.sbt index 9e875428..68354aa8 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "22.10.0-SNAPSHOT" +val releaseVersion = "22.12.0" val logbackVersion = "1.2.11" val opencensusVersion = "0.19.1" From a155b873329a9cb00384517b1eb3306962a487a1 Mon Sep 17 00:00:00 2001 From: Matthew Dickinson Date: Mon, 26 Dec 2022 04:22:12 +0000 Subject: [PATCH 396/409] Twitter-oss: update OSS libraries post-release to 23.3.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 23.3.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D1029024 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 50505902..39a9998d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) [![Maven Central](https://maven-badges.herokuapp.com/maven-central/com.twitter/twitter-server_2.12/badge.svg)](https://maven-badges.herokuapp.com/maven-central/com.twitter/twitter-server_2.12) diff --git a/build.sbt b/build.sbt index 68354aa8..62fdbb6c 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "22.12.0" +val releaseVersion = "23.3.0-SNAPSHOT" val logbackVersion = "1.2.11" val opencensusVersion = "0.19.1" From 33b3fb00635c4ab1102eb0c062cde6bb132d80c0 Mon Sep 17 00:00:00 2001 From: Matthew Dickinson Date: Fri, 22 Sep 2023 14:13:17 +0000 Subject: [PATCH 397/409] [Twitter-oss] bump release version [Twitter-oss] bump release version Differential Revision: https://phabricator.twitter.biz/D1100185 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 62fdbb6c..3e18af39 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "23.3.0-SNAPSHOT" +val releaseVersion = "23.9.0-SNAPSHOT" val logbackVersion = "1.2.11" val opencensusVersion = "0.19.1" From 0238c8526374231f3d585ac42356e8cec80e6cd8 Mon Sep 17 00:00:00 2001 From: andreoss Date: Tue, 31 Oct 2023 19:38:32 +0000 Subject: [PATCH 398/409] [twitter-server]: Portable shebang Problem: /bin/bash is not available on non-FHS distros, such as NixOS Solution: Replace /bin/bash with /usr/bin/env bash Result: Closes twitter/twitter-server#77 Differential Revision: https://phabricator.twitter.biz/D1107607 --- pushsite.bash | 2 +- sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pushsite.bash b/pushsite.bash index 291b56d5..c64a0098 100755 --- a/pushsite.bash +++ b/pushsite.bash @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -e diff --git a/sbt b/sbt index d08998b1..dbe5a021 100755 --- a/sbt +++ b/sbt @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eo pipefail From ec708f82ba97237596cdf074b4073405298bbd06 Mon Sep 17 00:00:00 2001 From: Matthew Dickinson Date: Wed, 1 Nov 2023 13:44:53 +0000 Subject: [PATCH 399/409] [Twitter-oss] bump release version [Twitter-oss] bump release version Differential Revision: https://phabricator.twitter.biz/D1107767 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 3e18af39..8e1bd0a4 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "23.9.0-SNAPSHOT" +val releaseVersion = "23.11.0-SNAPSHOT" val logbackVersion = "1.2.11" val opencensusVersion = "0.19.1" From c0e636d50752ae97c608b18f474728b5e95bfd42 Mon Sep 17 00:00:00 2001 From: Matthew Dickinson Date: Wed, 1 Nov 2023 23:58:08 +0000 Subject: [PATCH 400/409] Cut version 23.11 Problem/Solution: Cut version 23.11 of: - util - scrooge - finagle - twitter-server - finatra Differential Revision: https://phabricator.twitter.biz/D1107956 --- CHANGELOG.rst | 5 +++++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 5cb3cac7..72ae9282 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +23.11.0 +------- + +No Changes + 22.12.0 ------- diff --git a/README.md b/README.md index 39a9998d..50505902 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) [![Maven Central](https://maven-badges.herokuapp.com/maven-central/com.twitter/twitter-server_2.12/badge.svg)](https://maven-badges.herokuapp.com/maven-central/com.twitter/twitter-server_2.12) diff --git a/build.sbt b/build.sbt index 8e1bd0a4..cc50eff0 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "23.11.0-SNAPSHOT" +val releaseVersion = "23.11.0" val logbackVersion = "1.2.11" val opencensusVersion = "0.19.1" From 9e3d2e6e9bf852b5335582e2d5cd218242f1aed5 Mon Sep 17 00:00:00 2001 From: Matthew Dickinson Date: Thu, 2 Nov 2023 09:22:49 +0000 Subject: [PATCH 401/409] update to snapshot 24.2.0-snapshot Problem/Solution: Cut snapshot 24.2.0 of: - util - scrooge - finagle - twitter-server - finatra Differential Revision: https://phabricator.twitter.biz/D1108018 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 50505902..39a9998d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) [![Maven Central](https://maven-badges.herokuapp.com/maven-central/com.twitter/twitter-server_2.12/badge.svg)](https://maven-badges.herokuapp.com/maven-central/com.twitter/twitter-server_2.12) diff --git a/build.sbt b/build.sbt index cc50eff0..eb0a1e63 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "23.11.0" +val releaseVersion = "24.2.0-SNAPSHOT" val logbackVersion = "1.2.11" val opencensusVersion = "0.19.1" From 12ff1d86ec470846821b8b2b81f95cf3fb0980d7 Mon Sep 17 00:00:00 2001 From: Anton Ivanov Date: Fri, 10 Nov 2023 14:51:21 +0000 Subject: [PATCH 402/409] [twitter.server] offload ShutdownHandler # Problem ShutdownHandler is executed on a netty IO thread and calls app.close which can easily block this thread. # Solution Offload execution to a thread. JIRA Issues: STOR-1705 Differential Revision: https://phabricator.twitter.biz/D1109579 --- .../server/handler/ShutdownHandler.scala | 30 ++++++++++++++++--- .../server/handler/ShutdownHandlerTest.scala | 27 +++++++++++++---- 2 files changed, 48 insertions(+), 9 deletions(-) diff --git a/server/src/main/scala/com/twitter/server/handler/ShutdownHandler.scala b/server/src/main/scala/com/twitter/server/handler/ShutdownHandler.scala index 0108611f..b8ab28c4 100644 --- a/server/src/main/scala/com/twitter/server/handler/ShutdownHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/ShutdownHandler.scala @@ -2,11 +2,18 @@ package com.twitter.server.handler import com.twitter.app.App import com.twitter.finagle.Service -import com.twitter.finagle.http.{Method, Request, Response, Status, Uri} +import com.twitter.finagle.http.Method +import com.twitter.finagle.http.Request +import com.twitter.finagle.http.Response +import com.twitter.finagle.http.Status +import com.twitter.finagle.http.Uri import com.twitter.io.Buf -import com.twitter.server.util.HttpUtils.{newOk, newResponse} +import com.twitter.server.util.HttpUtils.newOk +import com.twitter.server.util.HttpUtils.newResponse import com.twitter.util.logging.Logger -import com.twitter.util.{Duration, Future} +import com.twitter.util.Duration +import com.twitter.util.Future +import com.twitter.util.Local class ShutdownHandler(app: App) extends Service[Request, Response] { private[this] val log = Logger[ShutdownHandler] @@ -38,7 +45,22 @@ class ShutdownHandler(app: App) extends Service[Request, Response] { s"[${req.uri}] from ${req.remoteAddress.getHostAddress} " + s"quitting with grace period $grace" ) - app.close(grace) + + // Isolate a current thread (likely netty IO thread) + // from probable blocking calls inside app.close. + val ctx = Local.save() + new Thread( + () => + try { + Local.restore(ctx) + app.close(grace) + } catch { + case th: Throwable => + log.error("Exception when asking an app to close", th) + }, + "shutdown-handler" // thread name + ).start() + newOk("quitting\n") } else { log.info( diff --git a/server/src/test/scala/com/twitter/server/handler/ShutdownHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/ShutdownHandlerTest.scala index 2d353721..16e9f80a 100644 --- a/server/src/test/scala/com/twitter/server/handler/ShutdownHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/handler/ShutdownHandlerTest.scala @@ -1,10 +1,21 @@ package com.twitter.server.handler import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.http.{Method, Request, Status} +import com.twitter.finagle.http.Method +import com.twitter.finagle.http.Request +import com.twitter.finagle.http.Status import com.twitter.server.TwitterServer -import com.twitter.util.{Await, Awaitable, Closable, Duration, Future, Time} +import com.twitter.util.Await +import com.twitter.util.Awaitable +import com.twitter.util.Closable +import com.twitter.util.Duration +import com.twitter.util.Future +import com.twitter.util.Time +import org.scalatest.concurrent.Eventually.eventually +import org.scalatest.concurrent.PatienceConfiguration.Timeout import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.time.Seconds +import org.scalatest.time.Span class ShutdownHandlerTest extends AnyFunSuite { @@ -38,7 +49,9 @@ class ShutdownHandlerTest extends AnyFunSuite { val handler = new ShutdownHandler(closer) val rsp = await(handler(Request(Method.Post, "/foo"))) assert(rsp.status == Status.Ok) - assert(closer.closed) + eventually(Timeout(Span(5, Seconds))) { + assert(closer.closed) + } }) test("close without a grace period, but closer overrides default grace")( @@ -53,7 +66,9 @@ class ShutdownHandlerTest extends AnyFunSuite { val handler = new ShutdownHandler(closer) val rsp = await(handler(Request(Method.Post, "/foo"))) assert(rsp.status == Status.Ok) - assert(closer.closed) + eventually(Timeout(Span(5, Seconds))) { + assert(closer.closed) + } }) test("close with a grace period") { @@ -66,7 +81,9 @@ class ShutdownHandlerTest extends AnyFunSuite { val handler = new ShutdownHandler(closer) val rsp = await(handler(Request(Method.Post, "/foo?grace=" + grace.toString))) assert(rsp.status == Status.Ok) - assert(closer.closed) + eventually(Timeout(Span(5, Seconds))) { + assert(closer.closed) + } } test("fail when an invalid grace parameter is specified") { From 8e29ecda17bd5cf5d4b929bcdd02321d1f14ded7 Mon Sep 17 00:00:00 2001 From: Chetan Tutika Date: Thu, 2 May 2024 15:30:24 +0000 Subject: [PATCH 403/409] Twitter-oss: Prepare OSS libraries for release 24.2.0 Problem We want to release the next versions of our Twitter OSS libraries 24.2.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D1140781 --- CHANGELOG.rst | 5 +++++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 72ae9282..d2335b72 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +24.2.0 +------ + +No Changes + 23.11.0 ------- diff --git a/README.md b/README.md index 39a9998d..50505902 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) [![Maven Central](https://maven-badges.herokuapp.com/maven-central/com.twitter/twitter-server_2.12/badge.svg)](https://maven-badges.herokuapp.com/maven-central/com.twitter/twitter-server_2.12) diff --git a/build.sbt b/build.sbt index eb0a1e63..0dbb57be 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "24.2.0-SNAPSHOT" +val releaseVersion = "24.2.0" val logbackVersion = "1.2.11" val opencensusVersion = "0.19.1" From fbba9ae8cebea92a7f26928676084ca5a50b13a8 Mon Sep 17 00:00:00 2001 From: Chetan Tutika Date: Mon, 6 May 2024 18:03:23 +0000 Subject: [PATCH 404/409] Revert "Twitter-oss: Prepare OSS libraries for release 24.2.0" This reverts commit 996858fd593b43d101d3c546d5a9968dc5408855. There were issues with the release. Will be retrying it Differential Revision: https://phabricator.twitter.biz/D1141484 --- CHANGELOG.rst | 5 ----- README.md | 2 +- build.sbt | 2 +- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d2335b72..72ae9282 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,11 +7,6 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- -24.2.0 ------- - -No Changes - 23.11.0 ------- diff --git a/README.md b/README.md index 50505902..39a9998d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) [![Maven Central](https://maven-badges.herokuapp.com/maven-central/com.twitter/twitter-server_2.12/badge.svg)](https://maven-badges.herokuapp.com/maven-central/com.twitter/twitter-server_2.12) diff --git a/build.sbt b/build.sbt index 0dbb57be..eb0a1e63 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "24.2.0" +val releaseVersion = "24.2.0-SNAPSHOT" val logbackVersion = "1.2.11" val opencensusVersion = "0.19.1" From 9d658ac4741287137de52ccfbd80f4cf2f84e041 Mon Sep 17 00:00:00 2001 From: Chetan Tutika Date: Mon, 6 May 2024 20:21:10 +0000 Subject: [PATCH 405/409] Twitter-oss: Prepare OSS libraries for release 24.5.0 Problem We want to release the next versions of our Twitter OSS libraries 24.5.0 util scrooge finagle twitter-server finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D1141504 --- CHANGELOG.rst | 5 +++++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 72ae9282..d7e03e4a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +24.5.0 +------ + +No Changes + 23.11.0 ------- diff --git a/README.md b/README.md index 39a9998d..50505902 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) [![Maven Central](https://maven-badges.herokuapp.com/maven-central/com.twitter/twitter-server_2.12/badge.svg)](https://maven-badges.herokuapp.com/maven-central/com.twitter/twitter-server_2.12) diff --git a/build.sbt b/build.sbt index eb0a1e63..73383fad 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "24.2.0-SNAPSHOT" +val releaseVersion = "24.5.0" val logbackVersion = "1.2.11" val opencensusVersion = "0.19.1" From e958a8c517287cd1490316336b7b6e49d729d6f0 Mon Sep 17 00:00:00 2001 From: Chetan Tutika Date: Tue, 7 May 2024 19:44:32 +0000 Subject: [PATCH 406/409] Twitter-oss: update OSS libraries post-release to 24.8.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 24.8.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D1141538 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 50505902..39a9998d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TwitterServer -[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) +[![Build Status](https://github.com/twitter/twitter-server/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/twitter-server/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) [![Maven Central](https://maven-badges.herokuapp.com/maven-central/com.twitter/twitter-server_2.12/badge.svg)](https://maven-badges.herokuapp.com/maven-central/com.twitter/twitter-server_2.12) diff --git a/build.sbt b/build.sbt index 73383fad..407d338f 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "24.5.0" +val releaseVersion = "24.8.0-SNAPSHOT" val logbackVersion = "1.2.11" val opencensusVersion = "0.19.1" From f39efac663fd5d76b8d4af0fee66313c16f733c2 Mon Sep 17 00:00:00 2001 From: Michael Anderson Date: Fri, 17 May 2024 20:22:08 +0000 Subject: [PATCH 407/409] update admin sidebar styling to dark mode update admin sidebar styling to dark mode Differential Revision: https://phabricator.twitter.biz/D1143989 --- server/src/main/resources/twitter-server/css/index.css | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/src/main/resources/twitter-server/css/index.css b/server/src/main/resources/twitter-server/css/index.css index 6c92c601..203d460e 100644 --- a/server/src/main/resources/twitter-server/css/index.css +++ b/server/src/main/resources/twitter-server/css/index.css @@ -28,7 +28,7 @@ h1,h2,h3,h4,h5,h6 { width: 250px; overflow-y: auto; overflow-x: auto; - background-color: #428bca; + background-color: rgb(0,0,0); -webkit-transition: all 0.5s ease; -moz-transition: all 0.5s ease; -o-transition: all 0.5s ease; @@ -47,7 +47,7 @@ h1,h2,h3,h4,h5,h6 { left: 250px; width: 12px; height: 100%; - background-color: #428bca; + background-color: rgb(0,0,0); border-left: 1px solid rgba(0, 0, 0, 0.1); border-right: 1px solid rgba(0, 0, 0, 0.1); box-shadow: 0 10px 0 red; @@ -59,7 +59,7 @@ h1,h2,h3,h4,h5,h6 { } #toggle:hover { - background-color: #2972B1; + background-color: rgb(83, 83, 83); } #wrapper.toggled #toggle { @@ -93,11 +93,11 @@ nav li { } nav .selectable:hover:not(.selected) { - background-color: #2972B1; + background-color: rgb(83, 83, 83); } nav .selected { - background-color: #0F5897; + background-color: rgb(0, 0, 0); } nav li { From c1d75b89a1581152f1601f99dac4ee581027ef12 Mon Sep 17 00:00:00 2001 From: Benjamin Erichsen Date: Wed, 16 Jul 2025 17:21:47 +0000 Subject: [PATCH 408/409] twitter-server: Add a readiness endpoint to query for accepting traffic. Problem We need an endpoint where we can rely on the HTTP status code to tell us if the server is ready to accept traffic. Solution Add a new endpoint called /ready that returns a 503 when the server is warming up and a 200 when the server finishes warming up and is ready for traffic. Differential Revision: https://phabricator.twitter.biz/D1240266 --- .../scala/com/twitter/server/Lifecycle.scala | 10 +++++++--- .../server/handler/ReadinessHandler.scala | 20 +++++++++++++++++++ 2 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 server/src/main/scala/com/twitter/server/handler/ReadinessHandler.scala diff --git a/server/src/main/scala/com/twitter/server/Lifecycle.scala b/server/src/main/scala/com/twitter/server/Lifecycle.scala index bc1ac849..0f890a17 100644 --- a/server/src/main/scala/com/twitter/server/Lifecycle.scala +++ b/server/src/main/scala/com/twitter/server/Lifecycle.scala @@ -84,10 +84,12 @@ object Lifecycle { private[Lifecycle] object Warmup { /** - * Initialize warmup code. Ensures that the /health endpoint will not return on "OK" response. + * Initialize warmup code. Ensures that the /health endpoint will not return an "OK" response. */ - def initializeWarmup(): Unit = + def initializeWarmup(): Unit = { HttpMuxer.addHandler(Route("/health", new ReplyHandler("warming up\n"))) + HttpMuxer.addHandler(Route("/ready", new ReadinessHandler(isReady = false))) + } /** * Prebind warmup code. Used for warmup tasks that we want to run before we @@ -98,8 +100,10 @@ object Lifecycle { /** * The service is bound to a port and warmed up, announce health. */ - def warmupComplete(): Unit = + def warmupComplete(): Unit = { HttpMuxer.addHandler(Route("/health", new ReplyHandler("OK\n"))) + HttpMuxer.addHandler(Route("/ready", new ReadinessHandler(isReady = true))) + } } diff --git a/server/src/main/scala/com/twitter/server/handler/ReadinessHandler.scala b/server/src/main/scala/com/twitter/server/handler/ReadinessHandler.scala new file mode 100644 index 00000000..84a71666 --- /dev/null +++ b/server/src/main/scala/com/twitter/server/handler/ReadinessHandler.scala @@ -0,0 +1,20 @@ +package com.twitter.server.handler + +import com.twitter.finagle.Service +import com.twitter.finagle.http.Request +import com.twitter.finagle.http.Response +import com.twitter.finagle.http.Status +import com.twitter.io.Buf +import com.twitter.server.util.HttpUtils.newOk +import com.twitter.server.util.HttpUtils.newResponse +import com.twitter.util.Future + +class ReadinessHandler(isReady: Boolean) extends Service[Request, Response] { + def apply(req: Request): Future[Response] = + if (isReady) newOk("Ready") + else + newResponse( + status = Status.ServiceUnavailable, + contentType = "text/plain;charset=UTF-8", + content = Buf.Utf8("Not ready. Warming up.")) +} From 24e15a4b0110e9807ed104c1ae2097ae8769d0db Mon Sep 17 00:00:00 2001 From: Benjamin Erichsen Date: Fri, 13 Feb 2026 19:02:45 +0000 Subject: [PATCH 409/409] Add a ready endpoint to the default Twitter Server Add a ready endpoint to the default Twitter Server, which signals the server is ready to accept traffic. Differential Revision: https://phabricator.twitter.biz/D1303727 --- server/src/main/scala/com/twitter/server/Lifecycle.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/scala/com/twitter/server/Lifecycle.scala b/server/src/main/scala/com/twitter/server/Lifecycle.scala index 0f890a17..06ac1acb 100644 --- a/server/src/main/scala/com/twitter/server/Lifecycle.scala +++ b/server/src/main/scala/com/twitter/server/Lifecycle.scala @@ -37,7 +37,7 @@ trait Lifecycle { self: TwitterServer => index = Some(RouteIndex(alias = "Health", group = group)) ) ) - + HttpMuxer.addHandler(Route("/ready", new ReadinessHandler(isReady = true))) } object promoteBeforeServing