{"templateId":"markdown","sharedDataIds":{"sidebar":"sidebar-sidebars.yaml"},"props":{"metadata":{"markdoc":{"tagList":["admonition","Steps","Step"]},"type":"markdown"},"seo":{"title":"Implementing Kafka event streaming","description":"Welcome to the Upvest API documentation. Here you will find all the information you need to integrate with our API.","siteUrl":"https://docs.upvest.co/","image":"/assets/upvest-logo-card.ee25ccc59849324b0b73151a5c972c87639419556344db0b6dcf518151cb4c3c.925ff8b2.png","keywords":"documentation, api","lang":"en-US","meta":[{"name":"description","content":"Welcome to the Upvest API documentation. Here you will find all the information you need to integrate with our API."},{"name":"image","content":"/assets/upvest-logo-card.ee25ccc59849324b0b73151a5c972c87639419556344db0b6dcf518151cb4c3c.925ff8b2.png"},{"name":"twitter:card","content":"summary_large_image"},{"name":"twitter:title","content":"Welcome to the Upvest API Documentation"},{"name":"twitter:image","content":"/assets/upvest-logo-card.ee25ccc59849324b0b73151a5c972c87639419556344db0b6dcf518151cb4c3c.925ff8b2.png"},{"name":"twitter:description","content":"Welcome to the Upvest API documentation. Here you will find all the information you need to integrate with our API."},{"name":"og:url","content":"https://docs.upvest.co/"},{"name":"og:title","content":"Welcome to the Upvest API Documentation"},{"name":"og:description","content":"Welcome to the Upvest API documentation. Here you will find all the information you need to integrate with our API."},{"name":"og:image","content":"/assets/upvest-logo-card.ee25ccc59849324b0b73151a5c972c87639419556344db0b6dcf518151cb4c3c.925ff8b2.png"}]},"dynamicMarkdocComponents":[],"compilationErrors":[],"ast":{"$$mdtype":"Tag","name":"article","attributes":{},"children":[{"$$mdtype":"Tag","name":"Heading","attributes":{"level":1,"id":"implementing-kafka-event-streaming","__idx":0},"children":["Implementing Kafka event streaming"]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"prerequisites","__idx":1},"children":["Prerequisites"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Upvest provisions access for you. Before you connect, make sure you have received the following from your Upvest contact:"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Bootstrap server."]}," The ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["hostname"]}," and ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["port"]}," your Kafka client connects to."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Username and password."]}," Your credentials for the cluster."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Simple Authentication and Security Layer (SASL) mechanism."]}," The authentication mechanism to configure on your client."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"connection-parameters","__idx":2},"children":["Connection parameters"]},{"$$mdtype":"Tag","name":"div","attributes":{"className":"md-table-wrapper"},"children":[{"$$mdtype":"Tag","name":"table","attributes":{"className":"md"},"children":[{"$$mdtype":"Tag","name":"thead","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Parameter"},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Parameter"]}]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Value"},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Value"]}]}]}]},{"$$mdtype":"Tag","name":"tbody","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Bootstrap server"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["<BOOTSTRAP_SERVER>"]}]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Security protocol"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["SASL_SSL"]}]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["SASL mechanism"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Provided by Upvest"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Username"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["<USERNAME>"]}]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Password"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["<PASSWORD>"]}]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Transport Layer Security (TLS)"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Standard system Certificate Authority (CA) bundle, no custom certificate required"]}]}]}]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Connections are authenticated over TLS using a publicly trusted certificate authority, so you do not need any custom certificate authority configuration. Use the SASL mechanism, username, and password exactly as provided by Upvest."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["You can connect using any standard Kafka client to complete the connection."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"topics","__idx":3},"children":["Topics"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Your topics are exclusive to your organisation. Upvest provisions two types of topics:"]},{"$$mdtype":"Tag","name":"div","attributes":{"className":"md-table-wrapper"},"children":[{"$$mdtype":"Tag","name":"table","attributes":{"className":"md"},"children":[{"$$mdtype":"Tag","name":"thead","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Topic"},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Topic"]}]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Contents"},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Contents"]}]}]}]},{"$$mdtype":"Tag","name":"tbody","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["Notifications"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["All event types you would otherwise receive over webhooks"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["Prices"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Price update events"]}]}]}]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Upvest gives you the exact topic names to use. Use each name as provided, without adding a prefix or suffix."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"message-format","__idx":4},"children":["Message format"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Each message value is a single event, encoded as a JSON UTF-8 string. The event has the same structure as the events delivered over webhooks, so you can reuse the same parsing logic."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Each event contains the following fields:"]},{"$$mdtype":"Tag","name":"div","attributes":{"className":"md-table-wrapper"},"children":[{"$$mdtype":"Tag","name":"table","attributes":{"className":"md"},"children":[{"$$mdtype":"Tag","name":"thead","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Field name"},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Field name"]}]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Type"},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Type"]}]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Description"},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Description"]}]}]}]},{"$$mdtype":"Tag","name":"tbody","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["id"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["String (uuid)"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["The unique ID of the event."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["created_at"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["String"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["An ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://datatracker.ietf.org/doc/html/rfc3339"},"children":["RFC 3339"]}," timestamp indicating the time at which the event occurred."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["type"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["String"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["The event type. See the ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/products/omnibus/getting_started/implementing_webhooks/webhooks_categories_types"},"children":["Event categories & types"]}," section for the full list."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["object"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["JSON object"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["The event-specific payload, identical to the ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["object"]}," documented for the same event."]}]}]}]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The event payload contains the exact same fields as the related webhook. The ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["object"]}," you receive on a topic matches the ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["object"]}," in the webhook for that event type."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The only difference is the envelope. A webhook request bundles several events under a top-level ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["payload"]}," array; Kafka delivers one event per message, so there is no ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["payload"]}," wrapper to unpack."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Each message also carries Kafka headers for routing without parsing the body:"]},{"$$mdtype":"Tag","name":"div","attributes":{"className":"md-table-wrapper"},"children":[{"$$mdtype":"Tag","name":"table","attributes":{"className":"md"},"children":[{"$$mdtype":"Tag","name":"thead","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Header"},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Header"]}]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Value"},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Value"]}]}]}]},{"$$mdtype":"Tag","name":"tbody","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["event-id"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["The event ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["id"]}]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["event-type"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["The event ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["type"]}]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["content-type"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["application/json"]}]}]}]}]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Kafka preserves the order of messages within a partition. However, the message order is not guaranteed across partitions, so you should design your consumer to handle each event independently, where possible."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Example message value"]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["A single ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["USER.CREATED"]}," event, as delivered on a topic. For the exact fields of this event, see the ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/api/users/user_event"},"children":["user event reference"]},":"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"json","header":{"controls":{"copy":{}}},"source":"{\n    \"id\": \"fbecea50-2f35-4969-96af-342271da9eca\",\n    \"created_at\": \"2021-07-21T14:10:00.00Z\",\n    \"type\": \"USER.CREATED\",\n    \"object\": {\n        \"id\": \"83d83ec2-d2ca-49ff-bbea-b92b5c3be202\",\n        \"type\": \"USER\"\n    }\n}\n","lang":"json"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Parse the message value as JSON before you access any field, and handle unknown fields gracefully, because new fields can be added in non-breaking releases."]},{"$$mdtype":"Tag","name":"Admonition","attributes":{"type":"warning"},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["If a message cannot be parsed as valid JSON, treat it as a processing error and route it to your dead-letter queue (DLQ) or alerting pipeline. Do not silently discard malformed messages."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"consumer-groups","__idx":5},"children":["Consumer groups"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Use a consumer group ID that is unique to your application. A clear convention is to name it after the service or use case that consumes the events, for example ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["notifications-consumer"]}," or ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["reconciliation"]},". Run multiple instances with the same group ID to share the partition workload automatically."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"testing-connectivity","__idx":6},"children":["Testing connectivity"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Before you integrate, verify the connection with ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["kcat"]},", a command-line Kafka client. Install it on your platform, then run the commands below, filling in the SASL mechanism Upvest provided."]},{"$$mdtype":"Tag","name":"Steps","attributes":{},"children":[{"$$mdtype":"Tag","name":"Step","attributes":{"title":"List topics to confirm authentication"},"children":[{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"kcat -b <BOOTSTRAP_SERVER> \\\n     -X security.protocol=SASL_SSL \\\n     -X sasl.mechanism=<SASL_MECHANISM> \\\n     -X sasl.username=<USERNAME> \\\n     -X sasl.password=<PASSWORD> \\\n     -L\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["A successful metadata response confirms that authentication works."]}]},{"$$mdtype":"Tag","name":"Step","attributes":{"title":"Consume from the beginning of a topic"},"children":[{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"kcat -b <BOOTSTRAP_SERVER> \\\n     -X security.protocol=SASL_SSL \\\n     -X sasl.mechanism=<SASL_MECHANISM> \\\n     -X sasl.username=<USERNAME> \\\n     -X sasl.password=<PASSWORD> \\\n     -C -t <TOPIC_NAME> -o beginning\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Each line is one event message value, in the JSON format shown above. If no messages appear, the topic may be empty. That is not a connectivity issue."]}]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"environments","__idx":7},"children":["Environments"]},{"$$mdtype":"Tag","name":"div","attributes":{"className":"md-table-wrapper"},"children":[{"$$mdtype":"Tag","name":"table","attributes":{"className":"md"},"children":[{"$$mdtype":"Tag","name":"thead","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Environment"},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Environment"]}]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Purpose"},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Purpose"]}]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Bootstrap server"},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Bootstrap server"]}]}]}]},{"$$mdtype":"Tag","name":"tbody","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Sandbox"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Integration testing with no real data"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Provided by Upvest"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Production"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Live data"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Provided after sandbox sign-off"]}]}]}]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Credentials are issued separately per environment. Sandbox credentials do not work against production, and production credentials do not work against sandbox."]}]},"headings":[{"value":"Implementing Kafka event streaming","id":"implementing-kafka-event-streaming","depth":1},{"value":"Prerequisites","id":"prerequisites","depth":2},{"value":"Connection parameters","id":"connection-parameters","depth":2},{"value":"Topics","id":"topics","depth":2},{"value":"Message format","id":"message-format","depth":2},{"value":"Consumer groups","id":"consumer-groups","depth":2},{"value":"Testing connectivity","id":"testing-connectivity","depth":2},{"value":"Environments","id":"environments","depth":2}],"frontmatter":{"seo":{"title":"Implementing Kafka event streaming"}},"lastModified":"2026-06-04T15:15:17.000Z","pagePropGetterError":{"message":"","name":""}},"slug":"/products/omnibus/getting_started/event_streaming/implementing_kafka","userData":{"isAuthenticated":false,"teams":["anonymous"]},"isPublic":true}