DSLs vs. Libraries: Evaluating Language Design in the GenAI Era

import pandas as pd
import json
import time
import logging
from typing import Dict, List, Any
from pathlib import Path

class DataPipeline:
def __init__(self, input_file: str, output_file: str):
self.input_file = input_file
self.output_file = output_file
self.logger = self._setup_logging()
self.processed_data = None

def _setup_logging(self) -> logging.Logger:
logging.basicConfig(level=logging.INFO)
        return logging.getLogger(__name__)

• All state variables and data flow are managed manually and each operation is executed when called.
• Operations execute in the order in which they are written and the developer has full access to intermediate results and execution path.

Section 2: Data Reading and Parsing

DSL (Apache Beam)

with beam.Pipeline(options=pipeline_options) as pipeline:
    parsed_data = (pipeline
                  | 'Read CSV' >> beam.io.ReadFromText('users.csv', skip_header_lines=1)
                  | 'Parse CSV' >> beam.Map(lambda line: line.split(','))
                  | 'Create Records' >> beam.Map(lambda fields: {
                      'name': fields[0],
                      'age': int(fields[1]),
                      'country': fields[2],
                       'active': fields[3].lower() == 'true'
                  }))

• The framework automatically splits files into chunks for parallel reading. ReadFromText creates a PTransform that abstracts file I/O.
• Data flows through transformations without materializing intermediate results and Beam infers data types and optimizes serialization

GPL (Pandas)

def read_csv_data(self) -> pd.DataFrame:
    """Read and validate CSV data"""
    try:
        df = pd.read_csv(self.input_file)
        self.logger.info(f"Read {len(df)} records from {self.input_file}")
        return df
    except FileNotFoundError:
        self.logger.error(f"File {self.input_file} not found")
        raise
    except pd.errors.EmptyDataError:
        self.logger.error("CSV file is empty")
        raise

def validate_data(self, df: pd.DataFrame) -> pd.DataFrame:
    """Validate and clean data"""
    required_columns = ['name', 'age', 'country', 'active']
    missing_cols = set(required_columns) - set(df.columns)
    if missing_cols:
        raise ValueError(f"Missing columns: {missing_cols}")
    
    # Handle missing values
    df = df.dropna(subset=required_columns)
    
    # Convert data types
    df['age'] = pd.to_numeric(df['age'], errors='coerce')
    df['active'] = df['active'].astype(str).str.lower() == 'true'
    
return df.dropna(subset=['age']

• The entire dataset is loaded into memory as DataFrame. pd.read_csv() directly invokes pandas C extension
• Operations execute in order, each producing intermediate results

Section 3: Data Filtering

DSL (Apache Beam)

filtered_data = (parsed_data
                | 'Filter Adults' >> beam.Filter(lambda user: user['age'] > 18)

• beam.Filter() creates ParDo transform with a predicate function. Filter predicates added to execution graph and is not executed immediately
• Each element is processed independently across multiple workers. Framework batches elements automatically for efficient processing

GPL (Pandas)

def filter_data(self, df: pd.DataFrame) -> pd.DataFrame:
    """Apply business logic filters"""
    initial_count = len(df)
    
    # Filter adults
    df_filtered = df[df['age'] > 18]
    self.logger.info(f"Filtered {initial_count - len(df_filtered)} minors")
    
    # Filter active users
    df_filtered = df_filtered[df_filtered['active'] == True]
    self.logger.info(f"Final filtered dataset: {len(df_filtered)} records")
    
    return df_filtered

• Pandas uses NumPy for efficient array operations. Each filter creates a new DataFrame in memory. df[condition] creates a boolean mask and applies it.
• Each filter operation executes immediately on full dataset

Section 4: Data Transformation

DSL (Apache Beam)

transformed_data = (filtered_data
                   | 'Transform Names' >> beam.Map(lambda user: {
                       **user, 'name': user['name'].upper()

• beam.Map() creates element-wise transformation. Lambda functions serialized for distributed execution
• Original data is not modified and only new elements are created.

GPL (Pandas)

def transform_data(self, df: pd.DataFrame) -> pd.DataFrame:
    """Apply transformations"""
    df_transformed = df.copy()
    df_transformed['name'] = df_transformed['name'].str.upper()
    self.logger.info("Applied name transformation")
return df_transformed

• Direct manipulation of DataFrame columns. Pandas applies function to the entire column at once.
• pandas provides optimized string operations via .str accessor

Section 5: Data Aggregation

DSL (Apache Beam)

aggregated_data = (transformed_data
                  | 'Key By Country' >> beam.Map(lambda user: (user['country'], user))
                  | 'Group By Country' >> beam.GroupByKey()
                  | 'Aggregate' >> beam.Map(lambda country_users: {
                      'country': country_users[0],
                      'users': list(country_users[1]),
                      'count': len(list(country_users[1]))
}))

• GroupByKey triggers a distributed shuffle across workers. Each group processed independently on different workers
• Map creates (key, value) pairs for grouping. Framework automatically partitions data by key for efficient grouping.

GPL (Pandas)

def aggregate_by_country(self, df: pd.DataFrame) -> List[Dict[str, Any]]:
    """Group and aggregate data by country"""
    grouped = df.groupby('country')
    
    results = []
    for country, group in grouped:
        country_data = {
            'country': country,
            'count': len(group),
            'users': group[['name', 'age']].to_dict('records')
        }
        results.append(country_data)
    
    self.logger.info(f"Aggregated data for {len(results)} countries")
return results

• df.groupby creates a GroupBy object with grouped indices. pandas uses hash table to group rows by key values
• Groups processed one at a time in single thread

This comparison shows significant variations in runtime features and development approaches. With Apache Beam, the pipeline is developed in just 28 lines as compared to 118 lines in Pandas which results in 76% reduction in code and significantly faster development cycles. The DSL's declarative nature reduces cyclomatic complexity by approximately 15 in the GPL implementation to just 5, making the codebase easier to understand and maintain. This simplicity is accompanied by horizontal scaling capabilities and streaming memory efficiency through lazy evaluation, where Beam processes data in chunks rather than loading entire datasets into memory like Pandas.

However, this efficiency comes with trade-offs in developer control and debugging capabilities. The GPL approach offers complete visibility into intermediate results and explicit state management, making it substantially easier to debug and customize beyond standard domain patterns. Beam handles error management through their framework with some limitations, but in Pandas error handling is done by manual programming but it also offers fine-grained control over exceptions. The learning curve is different because Apache Beam needs domain-specific knowledge about distributed processing concepts, and Pandas depends on general programming knowledge.

Another key difference is observed in the case of memory usage. Beam’s streaming model allows it to scale effortlessly with large datasets, while Pandas operates entirely in memory, which limits it to the capacity of a single machine. According to Syntax density analysis, Beam has 4.2 operations per 10 lines, while pandas scans only 1.8 operations, highlighting the need and optimization for DSL specialization for data pipeline tasks.

Embedded DSL Acting as a Middle Ground

Embedded DSLs combine the advantages of both languages. They reuse the host’s operators, types, and tooling and avoid custom parsers, lexers, or compilers, which drastically reduces the learning curve while retaining full library support. External DSLs, in contrast, require their own syntax and toolchain. Embedded DSLs also allow domain-specific optimization which is not possible in general-purpose languages. For example SQL EDSLs such as Scala’s Slick and Haskell’s Persistent embed type-safe queries directly in application logic.

While performing the same task, I can use query language to show how embedded DSLs and external DSLs differ from one another:

External DSL (Pure SQL)

SELECT customer_id, AVG(order_total) as avg_order
FROM orders
WHERE order_date > '2025-01-01'
GROUP BY customer_id
HAVING AVG(order_total) > 100;

Embedded DSL (Scala's Slick)

val query = orders
  .filter(_.orderDate > Date.valueOf("2025-01-01"))
  .groupBy(_.customerId)
  .map { case (customerId, group) => 
    (customerId, group.map(_.orderTotal).avg) 
  }
  .filter(_._2 > 100)

The GenAI Evolution Impacting the Implementation of DSL and GPL

The rise of large language models capable of understanding and generating code in multiple languages has significantly changed the way domain specific languages are built. Generative AI has reduced the barriers to DSL creation by automating traditionally labor-intensive tasks of language design and implementation. Modern AI systems can assist in tasks like parser design, semantic analysis, and compiler construction making it feasible for domain experts to create specialized languages without deep expertise in programming language theory.

There has been equal advancement in general-purpose languages with the introduction of GenAI. AI-powered tools like Copilot assist with code completion, error detection, and refactoring which makes complex GPL codebases more manageable and accessible. AI can also generate repetitive code with fixed patterns, recommend libraries, automate tests, optimize performance and even translate between different languages bridging the gap between high-level intent and low-level implementation. 

The financial technology sector provides notable examples of how DSLs have been successfully employed to model complex financial contracts and trading strategies. The amazing work by Simon Peyton Jones and Jean-Marc Eber on financial contract modeling demonstrates how domain-specific abstractions can capture essential business logic more naturally than general-purpose programming languages. Inspired by their approach coupled with the advancements in AI, I developed an expressive DSL specifically designed for trading scenarios, which is capable of clearly defining trading logic through timelines, conditions and actions for simplifying the coding complexity without sacrificing performance or safety.

Each workflow whether it is formulating trading questions, testing strategies, monitoring risks, or routing orders consistently follows a clear three-step process: Observe → Detect → React. Using this idea as the core knowledge the key design principles that emerged were:

1. Timelines as first‑class citizens: Model every input (ticks, candles, macros, PnL curves) as an Observable<T>stamped by event and ingest time.
2. Uniform composition: Support algebraic operations (map, filter, combineLatest, window) over timelines, so any workflow is just a DAG of transforms.
3. Name‑based resolver: Decouple syntax from implementation; each operator, data source, or action is identified by a string key resolved at runtime, enabling hot‑swapping and LLM‑driven stub generation.
4. Monoidal state: Actions produce diffs merged atomically, giving audit trails and safe side‑effects.

This abstraction keeps the size of the grammar (~30 tokens) constant while working across multiple levels of trading sophistication. As every dynamic part of the DSL is not accessed in any other way than by its symbolical name, a large language model can act as an on-demand code generator. When the engine first meets an unknown symbol, it emits a type-safe stub, continues execution with a harmless default, and immediately feeds that stub’s docstring plus sample I/O to the LLM. The model synthesises a concrete implementation, the hot-reloader swaps it into the running process, and subsequent ticks use the real logic—no grammar edits, no redeploys, zero downtime. In effect, GenAI turns our resolver into an infinite, self-extending standard library that grows exactly where traders push it next.

Below is a tiny proof‑of‑concept showing how one might define a 1-minute VWAP query in our DSL versus an equivalent Python library implementation.

DSL Usage (Pseudo‑JSON)

{
  "pipeline": [
    { "op": "Observable.CurrencyTicks", "params": { "symbol": "EURUSD" } },
    { "op": "Window",            "params": { "size": "1m", "type": "time" } },
    { "op": "Aggregate.VWAP",    "params": {} },
    { "op": "Action.Print",      "params": {} }
  ]
}

Equivalent Python Library (Pandas / AsyncIO)

import pandas as pd
import asyncio

async def stream_ticks(symbol, out_queue):
    async for tick in price_feed(symbol):
        out_queue.put_nowait(tick)

async def vwap(window_seconds=60):
    queue = asyncio.Queue()
    asyncio.create_task(stream_ticks("EURUSD", queue))
    buf = []
    start = None
    while True:
        tick = await queue.get()
        if start is None:
            start = tick.timestamp
        buf.append(tick)
        if tick.timestamp - start >= window_seconds:
            df = pd.DataFrame([{'price': t.price, 'volume': t.volume} for t in buf])
            vwap = (df.price * df.volume).sum() / df.volume.sum()
            print(f"VWAP: {vwap}")
buf.clear(); start = None

Coming to an End

Domain-Specific Languages DSLs are very useful if the application domain is a stable and mature and has clearly defined requirements within a limited scope. They provide concise, high-level, declarative syntax closely mapped to domain-specific tasks, significantly reducing repetitive coding and cognitive load. On the other hand General-Purpose Languages (GPLs) excel in scenarios where requirements are dynamic, multiple domains are covered, or the problem space is not clearly defined. They provide the flexibility required to adapt rapidly, offer Turing completeness and provide deep integration with any kind of APIs, databases and protocols. As AI capabilities develop over time, the primary consideration in choosing between DSLs and GPL libraries may shift from implementation concerns to questions of domain modeling and user experience. The trading platform DSL's ability to simplify complicated financial operations into a timeline algebra shows that regardless of the particular implementation technology used to make those abstractions a reality, the future belongs to methods that can elegantly abstract key domain patterns.