Merge pull request #821 from w3-bounty/feature/cloudflare-ai-gateway-clean

odilitime · web-flow · commit 49793e254819 · 2025-01-10T19:08:13.000-08:00
feat: Add Cloudflare AI Gateway support
diff --git a/.env.example b/.env.example
@@ -363,6 +363,12 @@ FLOW_ENDPOINT_URL=              # Default: https://mainnet.onflow.org
 INTERNET_COMPUTER_PRIVATE_KEY=
 INTERNET_COMPUTER_ADDRESS=
 
+
+#Cloudflare AI Gateway
+CLOUDFLARE_GW_ENABLED=        # Set to true to enable Cloudflare AI Gateway
+CLOUDFLARE_AI_ACCOUNT_ID=     # Cloudflare AI Account ID - found in the Cloudflare Dashboard under AI Gateway
+CLOUDFLARE_AI_GATEWAY_ID=     # Cloudflare AI Gateway ID - found in the Cloudflare Dashboard under AI Gateway
+
 # Aptos
 APTOS_PRIVATE_KEY=              # Aptos private key
 APTOS_NETWORK=                  # Must be one of mainnet, testnet
diff --git a/docs/docs/guides/configuration.md b/docs/docs/guides/configuration.md
@@ -71,6 +71,59 @@ HEURIST_API_KEY=
 # Livepeer Settings
 LIVEPEER_GATEWAY_URL=
 ```
+
+### Cloudflare AI Gateway Integration
+
+Eliza supports routing API calls through [Cloudflare AI Gateway](https://developers.cloudflare.com/ai-gateway/), which provides several benefits:
+
+- Detailed analytics and monitoring of message traffic and response times
+- Cost optimization through request caching and usage tracking across providers
+- Improved latency through Cloudflare's global network
+- Comprehensive visibility into message content and token usage
+- Cost analysis and comparison between different AI providers
+- Usage patterns and trends visualization
+- Request/response logging for debugging and optimization
+
+To enable Cloudflare AI Gateway:
+
+```bash
+# Cloudflare AI Gateway Settings
+CLOUDFLARE_GW_ENABLED=true
+CLOUDFLARE_AI_ACCOUNT_ID=your-account-id
+CLOUDFLARE_AI_GATEWAY_ID=your-gateway-id
+```
+
+Supported providers through Cloudflare AI Gateway:
+- OpenAI
+- Anthropic
+- Groq
+
+When enabled, Eliza will automatically route requests through your Cloudflare AI Gateway endpoint. The gateway URL is constructed in the format:
+```
+https://gateway.ai.cloudflare.com/v1/${accountId}/${gatewayId}/${provider}
+```
+
+If the gateway configuration is incomplete or disabled, Eliza will fall back to direct API calls.
+
+```bash
+# Cloudflare AI Gateway Settings
+CLOUDFLARE_GW_ENABLED=true
+CLOUDFLARE_AI_ACCOUNT_ID=your-account-id
+CLOUDFLARE_AI_GATEWAY_ID=your-gateway-id
+```
+
+Supported providers through Cloudflare AI Gateway:
+- OpenAI
+- Anthropic
+- Groq
+
+When enabled, Eliza will automatically route requests through your Cloudflare AI Gateway endpoint. The gateway URL is constructed in the format:
+```
+https://gateway.ai.cloudflare.com/v1/${accountId}/${gatewayId}/${provider}
+```
+
+If the gateway configuration is incomplete or disabled, Eliza will fall back to direct API calls.
+
 ### Image Generation
 
 Configure image generation in your character file:
diff --git a/packages/core/src/generation.ts b/packages/core/src/generation.ts
@@ -163,6 +163,50 @@ async function truncateTiktoken(
     }
 }
 
+/**
+ * Gets the Cloudflare Gateway base URL for a specific provider if enabled
+ * @param runtime The runtime environment
+ * @param provider The model provider name
+ * @returns The Cloudflare Gateway base URL if enabled, undefined otherwise
+ */
+function getCloudflareGatewayBaseURL(runtime: IAgentRuntime, provider: string): string | undefined {
+    const isCloudflareEnabled = runtime.getSetting("CLOUDFLARE_GW_ENABLED") === "true";
+    const cloudflareAccountId = runtime.getSetting("CLOUDFLARE_AI_ACCOUNT_ID");
+    const cloudflareGatewayId = runtime.getSetting("CLOUDFLARE_AI_GATEWAY_ID");
+
+    elizaLogger.debug("Cloudflare Gateway Configuration:", {
+        isEnabled: isCloudflareEnabled,
+        hasAccountId: !!cloudflareAccountId,
+        hasGatewayId: !!cloudflareGatewayId,
+        provider: provider
+    });
+
+    if (!isCloudflareEnabled) {
+        elizaLogger.debug("Cloudflare Gateway is not enabled");
+        return undefined;
+    }
+
+    if (!cloudflareAccountId) {
+        elizaLogger.warn("Cloudflare Gateway is enabled but CLOUDFLARE_AI_ACCOUNT_ID is not set");
+        return undefined;
+    }
+
+    if (!cloudflareGatewayId) {
+        elizaLogger.warn("Cloudflare Gateway is enabled but CLOUDFLARE_AI_GATEWAY_ID is not set");
+        return undefined;
+    }
+
+    const baseURL = `https://gateway.ai.cloudflare.com/v1/${cloudflareAccountId}/${cloudflareGatewayId}/${provider.toLowerCase()}`;
+    elizaLogger.info("Using Cloudflare Gateway:", {
+        provider,
+        baseURL,
+        accountId: cloudflareAccountId,
+        gatewayId: cloudflareGatewayId
+    });
+
+    return baseURL;
+}
+
 /**
  * Send a message to the model for a text generateText - receive a string back and parse how you'd like
  * @param opts - The options for the generateText request.
@@ -242,6 +286,16 @@ export async function generateText({
     }
 
     const provider = runtime.modelProvider;
+    elizaLogger.debug("Provider settings:", {
+        provider,
+        hasRuntime: !!runtime,
+        runtimeSettings: {
+            CLOUDFLARE_GW_ENABLED: runtime.getSetting("CLOUDFLARE_GW_ENABLED"),
+            CLOUDFLARE_AI_ACCOUNT_ID: runtime.getSetting("CLOUDFLARE_AI_ACCOUNT_ID"),
+            CLOUDFLARE_AI_GATEWAY_ID: runtime.getSetting("CLOUDFLARE_AI_GATEWAY_ID")
+        }
+    });
+
     const endpoint =
         runtime.character.modelEndpointOverride || getEndpoint(provider);
     const modelSettings = getModelSettings(runtime.modelProvider, modelClass);
@@ -356,13 +410,17 @@ export async function generateText({
             case ModelProviderName.LLAMACLOUD:
             case ModelProviderName.NANOGPT:
             case ModelProviderName.HYPERBOLIC:
+            case ModelProviderName.TOGETHER:
             case ModelProviderName.NINETEEN_AI:
             case ModelProviderName.TOGETHER:
             case ModelProviderName.AKASH_CHAT_API: {
-                elizaLogger.debug("Initializing OpenAI model.");
+                elizaLogger.debug("Initializing OpenAI model with Cloudflare check");
+                const baseURL = getCloudflareGatewayBaseURL(runtime, 'openai') || endpoint;
+
+                //elizaLogger.debug("OpenAI baseURL result:", { baseURL });
                 const openai = createOpenAI({
                     apiKey,
-                    baseURL: endpoint,
+                    baseURL,
                     fetch: runtime.fetch,
                 });
 
@@ -430,10 +488,7 @@ export async function generateText({
                 const { text: openaiResponse } = await aiGenerateText({
                     model: openai.languageModel(model),
                     prompt: context,
-                    system:
-                        runtime.character.system ??
-                        settings.SYSTEM_PROMPT ??
-                        undefined,
+                    system: runtime.character.system ?? settings.SYSTEM_PROMPT ?? undefined,
                     temperature: temperature,
                     maxTokens: max_response_length,
                     frequencyPenalty: frequency_penalty,
@@ -474,13 +529,11 @@ export async function generateText({
             }
 
             case ModelProviderName.ANTHROPIC: {
-                elizaLogger.debug("Initializing Anthropic model.");
-
-                const anthropic = createAnthropic({
-                    apiKey,
-                    fetch: runtime.fetch,
-                });
+                elizaLogger.debug("Initializing Anthropic model with Cloudflare check");
+                const baseURL = getCloudflareGatewayBaseURL(runtime, 'anthropic') || "https://api.anthropic.com/v1";
+                elizaLogger.debug("Anthropic baseURL result:", { baseURL });
 
+                const anthropic = createAnthropic({ apiKey, baseURL, fetch: runtime.fetch });
                 const { text: anthropicResponse } = await aiGenerateText({
                     model: anthropic.languageModel(model),
                     prompt: context,
@@ -568,26 +621,30 @@ export async function generateText({
             }
 
             case ModelProviderName.GROQ: {
-                const groq = createGroq({ apiKey, fetch: runtime.fetch });
+                elizaLogger.debug("Initializing Groq model with Cloudflare check");
+                const baseURL = getCloudflareGatewayBaseURL(runtime, 'groq');
+                elizaLogger.debug("Groq baseURL result:", { baseURL });
+                const groq = createGroq({ apiKey, fetch: runtime.fetch, baseURL });
 
                 const { text: groqResponse } = await aiGenerateText({
                     model: groq.languageModel(model),
                     prompt: context,
-                    temperature: temperature,
+                    temperature,
                     system:
                         runtime.character.system ??
                         settings.SYSTEM_PROMPT ??
                         undefined,
-                    tools: tools,
+                    tools,
                     onStepFinish: onStepFinish,
-                    maxSteps: maxSteps,
+                    maxSteps,
                     maxTokens: max_response_length,
                     frequencyPenalty: frequency_penalty,
                     presencePenalty: presence_penalty,
-                    experimental_telemetry: experimental_telemetry,
+                    experimental_telemetry,
                 });
 
                 response = groqResponse;
+                elizaLogger.debug("Received response from Groq model.");
                 break;
             }
 
@@ -1833,8 +1890,10 @@ async function handleOpenAI({
     schemaDescription,
     mode = "json",
     modelOptions,
+    provider,
+    runtime,
 }: ProviderOptions): Promise<GenerateObjectResult<unknown>> {
-    const baseURL = models.openai.endpoint || undefined;
+    const baseURL = getCloudflareGatewayBaseURL(runtime, 'openai') || models.openai.endpoint;
     const openai = createOpenAI({ apiKey, baseURL });
     return await aiGenerateObject({
         model: openai.languageModel(model),
@@ -1860,8 +1919,13 @@ async function handleAnthropic({
     schemaDescription,
     mode = "json",
     modelOptions,
+    runtime,
 }: ProviderOptions): Promise<GenerateObjectResult<unknown>> {
-    const anthropic = createAnthropic({ apiKey });
+    elizaLogger.debug("Handling Anthropic request with Cloudflare check");
+    const baseURL = getCloudflareGatewayBaseURL(runtime, 'anthropic');
+    elizaLogger.debug("Anthropic handleAnthropic baseURL:", { baseURL });
+
+    const anthropic = createAnthropic({ apiKey, baseURL });
     return await aiGenerateObject({
         model: anthropic.languageModel(model),
         schema,
@@ -1912,8 +1976,13 @@ async function handleGroq({
     schemaDescription,
     mode = "json",
     modelOptions,
+    runtime,
 }: ProviderOptions): Promise<GenerateObjectResult<unknown>> {
-    const groq = createGroq({ apiKey });
+    elizaLogger.debug("Handling Groq request with Cloudflare check");
+    const baseURL = getCloudflareGatewayBaseURL(runtime, 'groq');
+    elizaLogger.debug("Groq handleGroq baseURL:", { baseURL });
+
+    const groq = createGroq({ apiKey, baseURL });
     return await aiGenerateObject({
         model: groq.languageModel(model),
         schema,
