Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 65 additions & 0 deletions perf-tests/build.gradle
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
plugins {
id 'java'
}

java {
toolchain {
languageVersion = JavaLanguageVersion.of(17)
vendor = JvmVendorSpec.ADOPTIUM
}
}

repositories {
mavenCentral()
}

def langchainVersion = '1.8.0'

ext {
otelVersion = rootProject.ext.otelVersion
junitVersion = rootProject.ext.junitVersion
slf4jVersion = rootProject.ext.slf4jVersion
}

dependencies {
testImplementation project(":braintrust-sdk")
testImplementation project(":braintrust-sdk:instrumentation:langchain_1_8_0")
testImplementation project(":braintrust-java-agent:instrumenter")
testImplementation(testFixtures(project(":test-harness")))

testImplementation "io.opentelemetry:opentelemetry-api:${otelVersion}"
testImplementation "io.opentelemetry:opentelemetry-sdk:${otelVersion}"
testImplementation "io.opentelemetry:opentelemetry-sdk-trace:${otelVersion}"
testImplementation "io.opentelemetry:opentelemetry-sdk-logs:${otelVersion}"
testImplementation "io.opentelemetry:opentelemetry-sdk-metrics:${otelVersion}"

testImplementation "dev.langchain4j:langchain4j:${langchainVersion}"
testImplementation "dev.langchain4j:langchain4j-http-client:${langchainVersion}"
testImplementation "dev.langchain4j:langchain4j-open-ai:${langchainVersion}"

testImplementation 'net.bytebuddy:byte-buddy-agent:1.17.5'

testImplementation "org.junit.jupiter:junit-jupiter:${junitVersion}"
testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
testRuntimeOnly "org.slf4j:slf4j-simple:${slf4jVersion}"
}

// Disable the default test task so perf tests don't run during ./gradlew test or check.
// Run explicitly with: ./gradlew :perf-tests:perfTest
test {
enabled = false
}

task perfTest(type: Test) {
useJUnitPlatform()
workingDir = rootProject.projectDir

// Disable JUnit's per-test timeout — perf tests can take a while
systemProperty 'junit.jupiter.execution.timeout.default', 'disabled'

testLogging {
events "passed", "skipped", "failed"
showStandardStreams = true
exceptionFormat "full"
}
}
27 changes: 27 additions & 0 deletions perf-tests/src/test/java/dev/braintrust/perf/PerfResult.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
package dev.braintrust.perf;

/**
* Captures the result of a single performance test run.
*
* @param config the configuration that produced this result
* @param payloadBytes total bytes of the OTLP HTTP request body captured at the wire
* @param spanCount number of spans that were exported (as observed by the server)
* @param requestCount number of HTTP requests received by the capture server
*/
public record PerfResult(PerfRunConfig config, long payloadBytes, int spanCount, int requestCount) {

/** Bytes per span (approximate). */
public double bytesPerSpan() {
return spanCount > 0 ? (double) payloadBytes / spanCount : 0;
}

public double payloadMB() {
return payloadBytes / (1024.0 * 1024.0);
}

public String summary() {
return String.format(
"[%s] %d request(s), %d span(s), %.3f MB total (%.1f bytes/span)",
config.name(), requestCount, spanCount, payloadMB(), bytesPerSpan());
}
}
24 changes: 24 additions & 0 deletions perf-tests/src/test/java/dev/braintrust/perf/PerfRunConfig.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
package dev.braintrust.perf;

/**
* Describes the configuration for a single performance test run.
*
* <p>Each field controls the shape of the multi-turn conversation that will be generated and
* exported. Add new fields here as you add new scenarios (e.g. tool use, streaming, etc.).
*
* @param name human-readable label for this configuration
* @param turns number of conversational turns (user messages sent to the AI service)
* @param includeImageAttachment whether to include an image attachment in the first user message
*/
public record PerfRunConfig(String name, int turns, boolean includeImageAttachment) {

/** A multi-turn conversation with an image attachment on the first turn. */
public static PerfRunConfig multiTurnWithAttachment() {
return new PerfRunConfig("multi-turn-with-attachment", 10, true);
}

/** A multi-turn conversation with text only (no attachments). */
public static PerfRunConfig multiTurnTextOnly() {
return new PerfRunConfig("multi-turn-text-only", 3, false);
}
}
Loading
Loading