#!/usr/bin/env python3 """Generate two ~64k token prompts for n-gram spec benchmarking. Usage: python3 scripts/generate-prompts.py Outputs: /prompt_code.txt — code-heavy (~64k tokens) /prompt_prose.txt — prose-heavy (~64k tokens) The prompts are designed to be structurally different so we can observe whether n-gram speculative decoding helps more on one type than the other. Code prompt: structured, algorithmic, repetitive patterns (good for n-gram). Prose prompt: natural language, varied vocabulary (poor match for n-gram). """ import os import sys import random # Rough token estimate: 1 token ≈ 4 chars for English/code TOKEN_CHAR_RATIO = 4.0 TARGET_TOKENS = 64000 TARGET_CHARS = int(TARGET_TOKENS * TOKEN_CHAR_RATIO) def generate_code_prompt(): """Generate a code-heavy prompt with structured, repetitive patterns.""" parts = [] # Section 1: Python algorithms (well-structured, good for n-gram matching) algorithms = [ ("Merge Sort", """ def merge_sort(arr): \"\"\"Standard merge sort implementation.\"\"\" if len(arr) <= 1: return arr mid = len(arr) // 2 left = merge_sort(arr[:mid]) right = merge_sort(arr[mid:]) return merge(left, right) def merge(left, right): \"\"\"Merge two sorted arrays.\"\"\" result = [] i = j = 0 while i < len(left) and j < len(right): if left[i] <= right[j]: result.append(left[i]) i += 1 else: result.append(right[j]) j += 1 result.extend(left[i:]) result.extend(right[j:]) return result # Test merge sort if __name__ == "__main__": import random test_data = [random.randint(1, 1000) for _ in range(1000)] sorted_data = merge_sort(test_data) assert sorted_data == sorted(test_data) print(f"Merge sort passed: {len(test_data)} items in {len(sorted_data)}") """), ("Binary Search", """ def binary_search(arr, target): \"\"\"Binary search in a sorted array.\"\"\" low, high = 0, len(arr) - 1 while low <= high: mid = (low + high) // 2 if arr[mid] == target: return mid elif arr[mid] < target: low = mid + 1 else: high = mid - 1 return -1 # Test binary search if __name__ == "__main__": import random test_data = sorted(random.sample(range(1, 10000), 500)) for target in [test_data[0], test_data[-1], test_data[250], -1]: idx = binary_search(test_data, target) if target in test_data: assert test_data[idx] == target else: assert idx == -1 print(f"Binary search passed: {len(test_data)} items") """), ("DFS/BFS", """ from collections import deque def bfs(graph, start): \"\"\"Breadth-first search.\"\"\" visited = set() queue = deque([start]) visited.add(start) order = [] while queue: node = queue.popleft() order.append(node) for neighbor in sorted(graph.get(node, [])): if neighbor not in visited: visited.add(neighbor) queue.append(neighbor) return order def dfs(graph, start, visited=None): \"\"\"Depth-first search (iterative).\"\"\" if visited is None: visited = set() stack = [start] order = [] while stack: node = stack.pop() if node not in visited: visited.add(node) order.append(node) for neighbor in reversed(sorted(graph.get(node, []))): if neighbor not in visited: stack.append(neighbor) return order # Test graph traversal if __name__ == "__main__": import random nodes = list(range(50)) graph = {n: random.sample(nodes, random.randint(1, 5)) for n in nodes} bfs_order = bfs(graph, 0) dfs_order = dfs(graph, 0) print(f"BFS visited {len(bfs_order)} nodes, DFS visited {len(dfs_order)} nodes") """), ] for name, code in algorithms: parts.append(f"# === {name} ===\n") parts.append(code) # Section 2: Large data generation (repetitive structure) parts.append("\n# === Data Generation ===\n") parts.append("import json\nimport csv\nimport hashlib\n\n") for i in range(200): parts.append(f"# Batch {i}\n") parts.append(f"record_{i} = {{\n") parts.append(f" 'id': {i},\n") parts.append(f" 'name': 'record_{i}',\n") parts.append(f" 'value': {i * 3.14159},\n") parts.append(f" 'hash': hashlib.md5(b'record_{i}').hexdigest(),\n") parts.append(f" 'nested': {{\n") for j in range(10): parts.append(f" 'key_{j}': {i * j},\n") parts.append(f" }}\n") parts.append(f"}}\n") parts.append(f"batch_{i}.append(record_{i})\n") # Section 3: Configuration schema (highly repetitive, great for n-gram) parts.append("\n# === Configuration Schema ===\n") for i in range(50): parts.append(f"schema_{i} = {{\n") parts.append(f" 'type': 'object',\n") parts.append(f" 'properties': {{\n") for j in range(20): parts.append(f" 'field_{j}': {{\n") parts.append(f" 'type': 'string' if {j} % 2 else 'integer',\n") parts.append(f" 'required': {j} < 10,\n") parts.append(f" 'default': '{i}_{j}',\n") parts.append(f" 'description': 'Field {j} of schema {i}',\n") parts.append(f" }},\n") parts.append(f" }},\n") parts.append(f" 'required': ['field_0', 'field_1', 'field_2'],\n") parts.append(f" 'additionalProperties': False,\n") parts.append(f"}}\n") # Section 4: SQL queries (structured, repetitive) parts.append("\n# === SQL Queries ===\n") for i in range(50): parts.append(f"-- Query batch {i}\n") parts.append(f"SELECT * FROM table_{i}\n") parts.append(f"WHERE col_{i} = {i}\n") parts.append(f"ORDER BY id\n") parts.append(f"LIMIT {i * 100};\n\n") parts.append(f"INSERT INTO table_{i} (id, name, value)\n") parts.append(f"VALUES ({i}, 'record_{i}', {i});\n\n") parts.append(f"UPDATE table_{i}\n") parts.append(f"SET value = {i} * 1.5\n") parts.append(f"WHERE id = {i};\n\n") # Section 5: Documentation (structured, repetitive) parts.append("\n# === Documentation ===\n") for i in range(50): parts.append(f"## Section {i}\n\n") parts.append(f"### Overview\n") parts.append(f"This section covers topic {i} in detail. The purpose is to\n") parts.append(f"document the behavior of the system for component {i}.\n\n") parts.append(f"### Implementation\n") parts.append(f"The implementation uses a modular approach with {i} subcomponents.\n") parts.append(f"Each subcomponent handles a specific aspect of the functionality.\n\n") parts.append(f"### Testing\n") parts.append(f"Component {i} is tested with {i * 10} test cases.\n") parts.append(f"All tests pass with 100% coverage.\n\n") parts.append(f"### References\n") parts.append(f"- Reference {i}.1: Component specification\n") parts.append(f"- Reference {i}.2: Implementation guide\n") parts.append(f"- Reference {i}.3: Test documentation\n\n") # Fill remaining content with more code blocks while sum(len(p) for p in parts) < TARGET_CHARS: parts.append(f"# Additional block\n") parts.append(f"def extra_{len(parts)}():\n") parts.append(f" \"\"\"Additional function {len(parts)}.\"\"\"\n") parts.append(f" x = {len(parts)}\n") parts.append(f" y = x * 2\n") parts.append(f" return x + y\n\n") return "\n".join(parts)[:TARGET_CHARS] def generate_prose_prompt(): """Generate a prose-heavy prompt with varied vocabulary.""" parts = [] vocabulary = [ # Theme 1: Historical narrative ("The Renaissance", [ "The dawn of the Renaissance brought profound transformation to European civilization.", "Artists like Leonardo da Vinci and Michelangelo pushed the boundaries of human expression.", "The invention of the printing press by Gutenberg revolutionized the spread of knowledge.", "Scholars in Florence and Venice debated philosophy, science, and the nature of the cosmos.", "The Medici family funded countless artists, scholars, and architects.", "The rediscovery of classical Greek and Roman texts ignited intellectual renewal.", "Architecture shifted from the vertical aspiration of Gothic to the proportional harmony of classical orders.", "Patronage became a symbol of civic virtue and political power.", "The humanist movement placed human experience and reason at the center of intellectual inquiry.", "Mathematics, anatomy, and natural philosophy advanced through empirical observation.", ]), # Theme 2: Scientific exposition ("Quantum Mechanics", [ "Quantum mechanics describes the behavior of matter and energy at atomic scales.", "The Schrödinger equation governs the evolution of quantum states over time.", "Wave-particle duality reveals that light and matter exhibit both wave and particle properties.", "The uncertainty principle limits the precision with which conjugate variables can be measured.", "Quantum entanglement creates correlations between particles that persist across any distance.", "The Copenhagen interpretation posits that measurement collapses the wave function.", "Superposition allows quantum systems to exist in multiple states simultaneously until measured.", "Quantum tunneling enables particles to traverse classically forbidden energy barriers.", "The double-slit experiment demonstrates interference patterns from single particles.", "Quantum computing leverages superposition and entanglement for computational advantage.", ]), # Theme 3: Philosophical discourse ("Ethics and Moral Reasoning", [ "Moral philosophy grapples with questions of right and wrong, virtue and vice.", "Utilitarianism judges actions by their consequences and the happiness they produce.", "Deontological ethics emphasizes duties and rules rather than outcomes.", "Virtue ethics focuses on character development and flourishing rather than specific actions.", "The social contract theory posits that moral obligations arise from collective agreement.", "Moral skepticism challenges the possibility of objective moral knowledge.", "The problem of evil questions the compatibility of an omnipotent, benevolent deity with suffering.", "Free will debates center on whether human choices are genuinely open or determined.", "Moral relativism holds that ethical standards are culture-dependent rather than universal.", "The trolley dilemma illustrates tensions between utilitarian and deontological reasoning.", ]), ] for theme_name, sentences in vocabulary: parts.append(f"## {theme_name}\n\n") for sentence in sentences: parts.append(f"{sentence} ") parts.append("\n\n") # Section 2: Extended narrative with varied styles parts.append("## Extended Narrative\n\n") narratives = [ "The city stretched endlessly beneath a sky the color of television tuned to a dead channel.", "Mountains rose like ancient sentinels, their peaks lost in clouds that swirled and danced.", "The ocean whispered secrets to the shore, each wave carrying fragments of distant lands.", "In the quiet library, dust motes drifted through shafts of amber light filtering through stained glass.", "The market bustled with vendors calling out prices, the air thick with spices and laughter.", "Silent forests stretched for miles, their canopies forming a green cathedral overhead.", "The old lighthouse stood solitary against the storm, its beam cutting through the darkness.", "Rivers carved paths through valleys over millennia, patient and relentless.", "The desert shimmered in the heat, mirages dancing on the horizon like phantom cities.", "Snow fell silently on the rooftops, transforming the town into a winter wonderland.", "The ancient ruins whispered stories of a civilization long forgotten by the modern world.", "Stars scattered across the night sky like diamonds on velvet, billions of them.", "The forest floor was carpeted with ferns and moss, damp and fragrant with earth.", "Clocks ticked away in the old tower, each chime echoing through the narrow streets.", "The garden bloomed in an explosion of color, roses, tulips, and lavender in perfect arrangement.", "The train rattled through the countryside, passengers gazing at fields of golden wheat.", "Mountains echoed with the calls of eagles circling on thermal currents above.", "The village square hosted a weekly market where farmers sold their harvest and artisans displayed their crafts.", "A lone wolf howled in the distance, its mournful cry carrying across the valley.", "The cathedral vaulted overhead, ribbed arches converging at a central keystone.", ] for narrative in narratives: parts.append(f"{narrative} ") # Section 3: Generate more content by varying the narratives extensions = [ " Scholars have debated the implications of this phenomenon for decades, producing conflicting interpretations.", " Local legends claim that those who listen carefully can hear voices from centuries past.", " Researchers continue to study the effects, though evidence remains inconclusive.", " The architecture reflects a synthesis of influences from multiple cultures and time periods.", " Travelers who venture far enough often lose their sense of direction and time.", " The tradition has been passed down through generations, each adding their own interpretation.", " Modern technology has made it possible to study these phenomena with unprecedented precision.", " The natural beauty of the area attracts visitors from around the world seeking tranquility.", " Archaeologists uncovered artifacts dating back thousands of years, revealing a rich history.", " The climate has changed significantly over the centuries, altering the landscape dramatically.", " Scientists proposed multiple hypotheses to explain the observed patterns, each with varying degrees of support.", " The cultural significance of the site extends far beyond its physical boundaries.", " Visitors report feeling a profound sense of peace and connection to something greater.", " The geological formation resulted from a combination of volcanic activity and erosion over millions of years.", " The literature from this period provides insight into the social and political conditions of the time.", " Artists found inspiration in the dramatic landscapes, producing works that captured their essence.", " The discovery challenged established theories and prompted a reevaluation of fundamental assumptions.", " The community worked together to preserve the site, recognizing its importance to their heritage.", " Researchers used advanced imaging techniques to reveal hidden structures beneath the surface.", " The story has been retold in countless forms, from oral traditions to modern cinema.", ] for extension in extensions: parts.append(extension + " ") # Section 4: More narrative variations parts.append("\n\n## Additional Observations\n\n") observations = [ "The interplay between light and shadow creates a dynamic visual experience that changes throughout the day.", "Historical records indicate that this practice dates back to the early medieval period.", "The geological composition of the region provides valuable insights into planetary formation processes.", "Cultural traditions in the area reflect a complex interweaving of indigenous and immigrant influences.", "The ecological balance of the ecosystem depends on a delicate network of symbiotic relationships.", "Technological advances have dramatically altered the way we understand and interact with the natural world.", "The linguistic diversity of the region is remarkable, with dozens of languages and dialects spoken.", "Astronomical observations from this location have contributed significantly to our understanding of the cosmos.", "The artistic movement that originated here influenced creative expression across multiple continents.", "Economic factors played a crucial role in shaping the political landscape of the era.", ] for obs in observations: parts.append(f"{obs} ") # Section 5: Extended descriptions with varied sentence structures parts.append("\n\n## Detailed Descriptions\n\n") descriptions = [ ("The coastal town of Portmore sat nestled between steep cliffs and a sheltered bay, its whitewashed houses cascading down to a harbor filled with fishing boats and pleasure craft. The smell of salt and fish hung in the air, mingling with the aroma of fresh bread from the bakery on Main Street. Every morning at dawn, the fishermen would set out, their boats cutting through the glassy water, returning by midday with their catch. The market would be alive with the sounds of haggling, the glint of silver scales in the sunlight, and the colorful piles of fish laid out on ice. In the afternoons, tourists would wander the narrow streets, browsing artisan shops and sampling local wines at small tavernas. The old fortification on the headland offered panoramic views of the coastline, stretching for miles in both directions, with the open sea beyond.", "Nestled in the foothills of the mountain range, the village of Eldermere had remained largely unchanged for centuries. Its cobblestone streets wound between timber-framed houses with thatched roofs, their gardens overflowing with herbs and flowers. The village center was dominated by a medieval church with a stone tower that had been added during the fourteenth century. The parish priest, Father Thomas, had served the community for over three decades, and the villagers trusted him implicitly. On Sundays, the church would be filled to capacity, and the choir would sing hymns that echoed through the vaulted ceiling. Outside, the hills were covered in pastures where sheep grazed contentedly, tended by shepherds who knew every animal by name. The seasonal rhythms of the village followed the agricultural calendar: planting in spring, harvest in autumn, and a brief period of rest in winter. Local festivals celebrated these transitions with music, dance, and feasting that brought the community together."), ] for desc in descriptions: parts.append(f"{desc} ") # Fill remaining to reach target while sum(len(p) for p in parts) < TARGET_CHARS: parts.append(f"{random.choice([ "The morning sun cast long shadows across the landscape, illuminating every detail with warm, golden light.", "In the silence that followed, the weight of what had been said hung heavy in the air between them.", "The old manuscript had survived centuries of conflict, fire, and neglect, its pages yellowed but still legible.", "Scientists had been studying the phenomenon for years, yet its underlying mechanisms remained poorly understood.", "The city's architecture told the story of its evolution, from medieval fortifications to modern skyscrapers.", "Every season brought its own character to the valley, transforming the landscape in subtle but dramatic ways.", "The debate raged on, scholars on all sides presenting arguments and counterarguments with fierce conviction.", "The ancient tree stood as a testament to endurance, its gnarled branches reaching skyward after thousands of years.", "Cultural exchange between neighboring regions had enriched both societies, leading to innovations in art and technology.", "The discovery of the artifact reshaped our understanding of the civilization that had created it.", ])} ") return "\n".join(parts)[:TARGET_CHARS] def main(): if len(sys.argv) < 2: print(f"Usage: {sys.argv[0]} ") sys.exit(1) output_dir = sys.argv[1] os.makedirs(output_dir, exist_ok=True) print(f"Generating prompts...") code_prompt = generate_code_prompt() prose_prompt = generate_prose_prompt() code_path = os.path.join(output_dir, "prompt_code.txt") prose_path = os.path.join(output_dir, "prompt_prose.txt") with open(code_path, "w") as f: f.write(code_prompt) with open(prose_path, "w") as f: f.write(prose_prompt) # Estimate token counts code_tokens = len(code_prompt) // TOKEN_CHAR_RATIO prose_tokens = len(prose_prompt) // TOKEN_CHAR_RATIO print(f"Code prompt: {code_tokens:.0f} estimated tokens ({len(code_prompt):,} bytes) → {code_path}") print(f"Prose prompt: {prose_tokens:.0f} estimated tokens ({len(prose_prompt):,} bytes) → {prose_path}") if __name__ == "__main__": main()