Rework telemetry code to support sending events to Clickhouse

Co-Authored-By: Max Brunsfeld <maxbrunsfeld@gmail.com>

Cargo.lock

@@ -1972,6 +1972,7 @@ version = "0.1.0"
 dependencies = [
  "aho-corasick",
  "anyhow",
+ "client",
  "clock",
  "collections",
  "context_menu",

crates/client/src/client.rs

@@ -17,7 +17,7 @@ use futures::{
 use gpui::{
     actions,
     platform::AppVersion,
-    serde_json::{self, Value},
+    serde_json::{self},
     AnyModelHandle, AnyWeakModelHandle, AnyWeakViewHandle, AppContext, AsyncAppContext, Entity,
     ModelHandle, Task, View, ViewContext, WeakViewHandle,
 };
@@ -27,7 +27,7 @@ use postage::watch;
 use rand::prelude::*;
 use rpc::proto::{AnyTypedEnvelope, EntityMessage, EnvelopedMessage, PeerId, RequestMessage};
 use serde::Deserialize;
-use settings::{Settings, TelemetrySettings};
+use settings::Settings;
 use std::{
     any::TypeId,
     collections::HashMap,
@@ -47,6 +47,7 @@ use util::http::HttpClient;
 use util::{ResultExt, TryFutureExt};
 
 pub use rpc::*;
+pub use telemetry::ClickhouseEvent;
 pub use user::*;
 
 lazy_static! {
@@ -736,7 +737,7 @@ impl Client {
             read_from_keychain = credentials.is_some();
             if read_from_keychain {
                 cx.read(|cx| {
-                    self.report_event(
+                    self.telemetry().report_mixpanel_event(
                         "read credentials from keychain",
                         Default::default(),
                         cx.global::<Settings>().telemetry(),
@@ -1116,7 +1117,7 @@ impl Client {
                 .context("failed to decrypt access token")?;
             platform.activate(true);
 
-            telemetry.report_event(
+            telemetry.report_mixpanel_event(
                 "authenticate with browser",
                 Default::default(),
                 metrics_enabled,
@@ -1338,30 +1339,8 @@ impl Client {
         }
     }
 
-    pub fn start_telemetry(&self) {
+    pub fn telemetry(&self) -> &Arc<Telemetry> {
-        self.telemetry.start();
+        &self.telemetry
-    }
-
-    pub fn report_event(
-        &self,
-        kind: &str,
-        properties: Value,
-        telemetry_settings: TelemetrySettings,
-    ) {
-        self.telemetry
-            .report_event(kind, properties.clone(), telemetry_settings);
-    }
-
-    pub fn telemetry_log_file_path(&self) -> Option<PathBuf> {
-        self.telemetry.log_file_path()
-    }
-
-    pub fn metrics_id(&self) -> Option<Arc<str>> {
-        self.telemetry.metrics_id()
-    }
-
-    pub fn is_staff(&self) -> Option<bool> {
-        self.telemetry.is_staff()
     }
 }
 

crates/client/src/telemetry.rs

@@ -1,3 +1,4 @@
+use crate::{ZED_SECRET_CLIENT_TOKEN, ZED_SERVER_URL};
 use db::kvp::KEY_VALUE_STORE;
 use gpui::{
     executor::Background,
@@ -35,20 +36,56 @@ struct TelemetryState {
     release_channel: Option<&'static str>,
     os_version: Option<Arc<str>>,
     os_name: &'static str,
-    queue: Vec<MixpanelEvent>,
+    mixpanel_events_queue: Vec<MixpanelEvent>, // Mixpanel mixed events - will hopefully die soon
-    next_event_id: usize,
+    clickhouse_events_queue: Vec<ClickhouseEventWrapper>,
-    flush_task: Option<Task<()>>,
+    next_mixpanel_event_id: usize,
+    flush_mixpanel_events_task: Option<Task<()>>,
+    flush_clickhouse_events_task: Option<Task<()>>,
     log_file: Option<NamedTempFile>,
     is_staff: Option<bool>,
 }
 
 const MIXPANEL_EVENTS_URL: &'static str = "https://api.mixpanel.com/track";
 const MIXPANEL_ENGAGE_URL: &'static str = "https://api.mixpanel.com/engage#profile-set";
+const CLICKHOUSE_EVENTS_URL_PATH: &'static str = "/api/events";
 
 lazy_static! {
     static ref MIXPANEL_TOKEN: Option<String> = std::env::var("ZED_MIXPANEL_TOKEN")
         .ok()
         .or_else(|| option_env!("ZED_MIXPANEL_TOKEN").map(|key| key.to_string()));
+    static ref CLICKHOUSE_EVENTS_URL: String =
+        format!("{}{}", *ZED_SERVER_URL, CLICKHOUSE_EVENTS_URL_PATH);
+}
+
+#[derive(Serialize, Debug)]
+struct ClickhouseEventRequestBody {
+    token: &'static str,
+    installation_id: Option<Arc<str>>,
+    app_version: Option<Arc<str>>,
+    os_name: &'static str,
+    os_version: Option<Arc<str>>,
+    release_channel: Option<&'static str>,
+    events: Vec<ClickhouseEventWrapper>,
+}
+
+#[derive(Serialize, Debug)]
+struct ClickhouseEventWrapper {
+    time: u128,
+    signed_in: bool,
+    #[serde(flatten)]
+    event: ClickhouseEvent,
+}
+
+#[derive(Serialize, Debug)]
+#[serde(tag = "type")]
+pub enum ClickhouseEvent {
+    Editor {
+        operation: &'static str,
+        file_extension: Option<String>,
+        vim_mode: bool,
+        copilot_enabled: bool,
+        copilot_enabled_for_language: bool,
+    },
 }
 
 #[derive(Serialize, Debug)]
@@ -121,9 +158,11 @@ impl Telemetry {
                 release_channel,
                 device_id: None,
                 metrics_id: None,
-                queue: Default::default(),
+                mixpanel_events_queue: Default::default(),
-                flush_task: Default::default(),
+                clickhouse_events_queue: Default::default(),
-                next_event_id: 0,
+                flush_mixpanel_events_task: Default::default(),
+                flush_clickhouse_events_task: Default::default(),
+                next_mixpanel_event_id: 0,
                 log_file: None,
                 is_staff: None,
             }),
@@ -168,15 +207,24 @@ impl Telemetry {
                     let device_id: Arc<str> = device_id.into();
                     let mut state = this.state.lock();
                     state.device_id = Some(device_id.clone());
-                    for event in &mut state.queue {
+
+                    for event in &mut state.mixpanel_events_queue {
                         event
                             .properties
                             .distinct_id
                             .get_or_insert_with(|| device_id.clone());
                     }
-                    if !state.queue.is_empty() {
+
-                        drop(state);
+                    let has_mixpanel_events = !state.mixpanel_events_queue.is_empty();
-                        this.flush();
+                    let has_clickhouse_events = !state.clickhouse_events_queue.is_empty();
+                    drop(state);
+
+                    if has_mixpanel_events {
+                        this.flush_mixpanel_events();
+                    }
+
+                    if has_clickhouse_events {
+                        this.flush_clickhouse_events();
                     }
 
                     anyhow::Ok(())
@@ -231,7 +279,42 @@ impl Telemetry {
         }
     }
 
-    pub fn report_event(
+    pub fn report_clickhouse_event(
+        self: &Arc<Self>,
+        event: ClickhouseEvent,
+        telemetry_settings: TelemetrySettings,
+    ) {
+        if !telemetry_settings.metrics() {
+            return;
+        }
+
+        let mut state = self.state.lock();
+        let signed_in = state.metrics_id.is_some();
+        state.clickhouse_events_queue.push(ClickhouseEventWrapper {
+            time: SystemTime::now()
+                .duration_since(UNIX_EPOCH)
+                .unwrap()
+                .as_millis(),
+            signed_in,
+            event,
+        });
+
+        if state.device_id.is_some() {
+            if state.mixpanel_events_queue.len() >= MAX_QUEUE_LEN {
+                drop(state);
+                self.flush_clickhouse_events();
+            } else {
+                let this = self.clone();
+                let executor = self.executor.clone();
+                state.flush_clickhouse_events_task = Some(self.executor.spawn(async move {
+                    executor.timer(DEBOUNCE_INTERVAL).await;
+                    this.flush_clickhouse_events();
+                }));
+            }
+        }
+    }
+
+    pub fn report_mixpanel_event(
         self: &Arc<Self>,
         kind: &str,
         properties: Value,
@@ -243,7 +326,7 @@ impl Telemetry {
 
         let mut state = self.state.lock();
         let event = MixpanelEvent {
-            event: kind.to_string(),
+            event: kind.into(),
             properties: MixpanelEventProperties {
                 token: "",
                 time: SystemTime::now()
@@ -251,7 +334,7 @@ impl Telemetry {
                     .unwrap()
                     .as_millis(),
                 distinct_id: state.device_id.clone(),
-                insert_id: post_inc(&mut state.next_event_id),
+                insert_id: post_inc(&mut state.next_mixpanel_event_id),
                 event_properties: if let Value::Object(properties) = properties {
                     Some(properties)
                 } else {
@@ -264,17 +347,17 @@ impl Telemetry {
                 signed_in: state.metrics_id.is_some(),
             },
         };
-        state.queue.push(event);
+        state.mixpanel_events_queue.push(event);
         if state.device_id.is_some() {
-            if state.queue.len() >= MAX_QUEUE_LEN {
+            if state.mixpanel_events_queue.len() >= MAX_QUEUE_LEN {
                 drop(state);
-                self.flush();
+                self.flush_mixpanel_events();
             } else {
                 let this = self.clone();
                 let executor = self.executor.clone();
-                state.flush_task = Some(self.executor.spawn(async move {
+                state.flush_mixpanel_events_task = Some(self.executor.spawn(async move {
                     executor.timer(DEBOUNCE_INTERVAL).await;
-                    this.flush();
+                    this.flush_mixpanel_events();
                 }));
             }
         }
@@ -288,10 +371,10 @@ impl Telemetry {
         self.state.lock().is_staff
     }
 
-    fn flush(self: &Arc<Self>) {
+    fn flush_mixpanel_events(self: &Arc<Self>) {
         let mut state = self.state.lock();
-        let mut events = mem::take(&mut state.queue);
+        let mut events = mem::take(&mut state.mixpanel_events_queue);
-        state.flush_task.take();
+        state.flush_mixpanel_events_task.take();
         drop(state);
 
         if let Some(token) = MIXPANEL_TOKEN.as_ref() {
@@ -325,4 +408,53 @@ impl Telemetry {
                 .detach();
         }
     }
+
+    fn flush_clickhouse_events(self: &Arc<Self>) {
+        let mut state = self.state.lock();
+        let mut events = mem::take(&mut state.clickhouse_events_queue);
+        state.flush_clickhouse_events_task.take();
+        drop(state);
+
+        let this = self.clone();
+        self.executor
+            .spawn(
+                async move {
+                    let mut json_bytes = Vec::new();
+
+                    if let Some(file) = &mut this.state.lock().log_file {
+                        let file = file.as_file_mut();
+                        for event in &mut events {
+                            json_bytes.clear();
+                            serde_json::to_writer(&mut json_bytes, event)?;
+                            file.write_all(&json_bytes)?;
+                            file.write(b"\n")?;
+                        }
+                    }
+
+                    {
+                        let state = this.state.lock();
+                        json_bytes.clear();
+                        serde_json::to_writer(
+                            &mut json_bytes,
+                            &ClickhouseEventRequestBody {
+                                token: ZED_SECRET_CLIENT_TOKEN,
+                                installation_id: state.device_id.clone(),
+                                app_version: state.app_version.clone(),
+                                os_name: state.os_name,
+                                os_version: state.os_version.clone(),
+                                release_channel: state.release_channel,
+                                events,
+                            },
+                        )?;
+                    }
+
+                    this.http_client
+                        .post_json(CLICKHOUSE_EVENTS_URL.as_str(), json_bytes.into())
+                        .await?;
+                    anyhow::Ok(())
+                }
+                .log_err(),
+            )
+            .detach();
+    }
 }

crates/editor/Cargo.toml

@@ -23,6 +23,7 @@ test-support = [
 ]
 
 [dependencies]
+client = { path = "../client" }
 clock = { path = "../clock" }
 copilot = { path = "../copilot" }
 db = { path = "../db" }

crates/editor/src/editor.rs

@@ -22,6 +22,7 @@ pub mod test;
 use aho_corasick::AhoCorasick;
 use anyhow::{anyhow, Result};
 use blink_manager::BlinkManager;
+use client::ClickhouseEvent;
 use clock::ReplicaId;
 use collections::{BTreeMap, Bound, HashMap, HashSet, VecDeque};
 use copilot::Copilot;
@@ -1295,7 +1296,7 @@ impl Editor {
             cx.set_global(ScrollbarAutoHide(should_auto_hide_scrollbars));
         }
 
-        this.report_event("open editor", cx);
+        this.report_editor_event("open", cx);
         this
     }
 
@@ -6819,7 +6820,7 @@ impl Editor {
             .collect()
     }
 
-    fn report_event(&self, name: &str, cx: &AppContext) {
+    fn report_editor_event(&self, name: &'static str, cx: &AppContext) {
         if let Some((project, file)) = self.project.as_ref().zip(
             self.buffer
                 .read(cx)
@@ -6831,11 +6832,28 @@ impl Editor {
             let extension = Path::new(file.file_name(cx))
                 .extension()
                 .and_then(|e| e.to_str());
-            project.read(cx).client().report_event(
+            let telemetry = project.read(cx).client().telemetry().clone();
-                name,
+            telemetry.report_mixpanel_event(
-                json!({ "File Extension": extension, "Vim Mode": settings.vim_mode  }),
+                match name {
+                    "open" => "open editor",
+                    "save" => "save editor",
+                    _ => name,
+                },
+                json!({ "File Extension": extension, "Vim Mode": settings.vim_mode, "In Clickhouse": true  }),
                 settings.telemetry(),
             );
+            let event = ClickhouseEvent::Editor {
+                file_extension: extension.map(ToString::to_string),
+                vim_mode: settings.vim_mode,
+                operation: name,
+                copilot_enabled: settings.features.copilot,
+                copilot_enabled_for_language: settings.show_copilot_suggestions(
+                    self.language_at(0, cx)
+                        .map(|language| language.name())
+                        .as_deref(),
+                ),
+            };
+            telemetry.report_clickhouse_event(event, settings.telemetry())
         }
     }
 

crates/editor/src/items.rs

@@ -636,7 +636,7 @@ impl Item for Editor {
         project: ModelHandle<Project>,
         cx: &mut ViewContext<Self>,
     ) -> Task<Result<()>> {
-        self.report_event("save editor", cx);
+        self.report_editor_event("save", cx);
         let format = self.perform_format(project.clone(), FormatTrigger::Save, cx);
         let buffers = self.buffer().clone().read(cx).all_buffers();
         cx.spawn(|_, mut cx| async move {

crates/feedback/src/feedback_editor.rs

@@ -164,8 +164,9 @@ impl FeedbackEditor {
     ) -> anyhow::Result<()> {
         let feedback_endpoint = format!("{}/api/feedback", *ZED_SERVER_URL);
 
-        let metrics_id = zed_client.metrics_id();
+        let telemetry = zed_client.telemetry();
-        let is_staff = zed_client.is_staff();
+        let metrics_id = telemetry.metrics_id();
+        let is_staff = telemetry.is_staff();
         let http_client = zed_client.http_client();
 
         let request = FeedbackRequestBody {

crates/zed/src/main.rs

@@ -172,8 +172,8 @@ fn main() {
         })
         .detach();
 
-        client.start_telemetry();
+        client.telemetry().start();
-        client.report_event(
+        client.telemetry().report_mixpanel_event(
             "start app",
             Default::default(),
             cx.global::<Settings>().telemetry(),

crates/zed/src/zed.rs

@@ -573,7 +573,7 @@ fn open_telemetry_log_file(
     workspace.with_local_workspace(&app_state.clone(), cx, move |_, cx| {
         cx.spawn(|workspace, mut cx| async move {
             async fn fetch_log_string(app_state: &Arc<AppState>) -> Option<String> {
-                let path = app_state.client.telemetry_log_file_path()?;
+                let path = app_state.client.telemetry().log_file_path()?;
                 app_state.fs.load(&path).await.log_err()
             }