Repository navigation
Expand file tree
/
Copy pathcode_implementation_workflow.py
More file actions
1143 lines (988 loc) · 45.4 KB
/
Copy pathcode_implementation_workflow.py
File metadata and controls
1143 lines (988 loc) · 45.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
"""
Paper Code Implementation Workflow — unified on the core AgentRunner kernel.
P0 unification (see DEEPCODE_V2_MASTER_PLAN.md):
- The former hand-rolled ``_pure_code_implementation_loop`` (800-iteration
while loop with a bespoke text protocol) is gone. The implementation
phase now runs on ``core.agent_runtime.runner.AgentRunner`` — the same
kernel every other agent uses — with domain behavior injected through
kernel seams (tool wrappers, ``AgentHook``, ``should_stop_callback``,
``injection_callback``).
- Tool schemas come from the MCP servers themselves (registered via the
compat agent, re-exposed under their bare names). The hand-maintained
copies in ``config/mcp_tool_definitions*.py`` are deleted.
- ``CodeImplementationWorkflowWithIndex`` is now just
``CodeImplementationWorkflow(enable_indexing=True)``: same class, same
kernel, indexed prompt + minimal tool surface.
Domain strategy (unchanged, deliberately):
- ``ConciseMemoryAgent`` clean-slate memory: after each ``write_file``
the history collapses to plan + knowledge base.
- ``CodeImplementationAgent`` keeps executing/tracking tool calls
(read_file → read_code_mem interception, write_file summaries).
- Completion is mechanical: the run ends when every file parsed from the
plan exists — never because the model claims it is done.
"""
import asyncio
import json
import logging
import os
import sys
import time
import uuid
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional
from core.agent_runtime.hook import AgentHook, AgentHookContext
from core.agent_runtime.runner import AgentRunner, AgentRunSpec
from core.agent_runtime.tools.alias import AliasedTool, build_aliased_registry
from core.agent_runtime.tools.registry import ToolRegistry
from core.harness.approval import TerminalApprover
from core.harness.permissions import PermissionMode
from core.harness.policy import build_permission_engine
from core.verification import discover_verification_commands, run_verification
# DeepCode-native compat layer (owns the MCP server lifecycle)
from core.compat import Agent, get_runtime
from core.llm_runtime import attach_workflow_llm, get_workflow_provider
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from prompts.code_prompts import ( # noqa: E402
GENERAL_CODE_IMPLEMENTATION_SYSTEM_PROMPT,
PURE_CODE_IMPLEMENTATION_SYSTEM_PROMPT_INDEX,
STRUCTURE_GENERATOR_PROMPT,
)
from utils.llm_utils import get_default_models # noqa: E402
from utils.loop_detector import LoopDetector, ProgressTracker # noqa: E402
from workflows.agents import CodeImplementationAgent # noqa: E402
from workflows.agents.memory_agent_concise import ConciseMemoryAgent # noqa: E402
# Model-visible tool surfaces. These mirror the curated lists the deleted
# ``config/mcp_tool_definitions*.py`` used to hardcode — but the schemas
# now come from the MCP servers themselves (single source of truth).
_STANDARD_TOOL_NAMES = [
"read_file",
"read_multiple_files",
"read_code_mem",
"write_file",
"write_multiple_files",
"execute_python",
"execute_bash",
"get_file_structure",
"search_code_references",
"get_indexes_overview",
"set_workspace",
]
_INDEXED_TOOL_NAMES = [
"write_file",
"search_code_references",
]
_READ_TOOL_NAMES = {"read_file", "read_code_mem"}
_MCP_SERVER_NAMES = [
"code-implementation",
"code-reference-indexer",
"document-segmentation",
]
_MAX_ITERATIONS = 800
_MAX_WALL_SECONDS = 7200 # 120 minutes (2 hours)
_EMERGENCY_TRIM_THRESHOLD = 50
_MAX_TOOL_RESULT_CHARS = 60_000
@dataclass
class _RunState:
"""Shared mutable state between the tool wrappers, hook and callbacks."""
memory_agent: ConciseMemoryAgent
code_tracker: CodeImplementationAgent
loop_detector: LoopDetector
progress_tracker: ProgressTracker
logger: logging.Logger
system_prompt: str
guidance: "_GuidanceTexts"
progress_callback: Optional[Callable] = None
start_time: float = field(default_factory=time.time)
max_wall_seconds: float = _MAX_WALL_SECONDS
iterations_done: int = 0
in_tools_phase: bool = False
last_finish_reason: Optional[str] = None
abort_reason: Optional[str] = None
# (status, reason) once a terminal condition is known.
run_status: Optional[tuple] = None
def emit_progress(self, message: str) -> None:
if self.progress_callback:
try:
self.progress_callback(85, message)
except Exception: # noqa: BLE001 - progress must never kill the run
self.logger.debug("progress_callback failed", exc_info=True)
@dataclass(frozen=True)
class _GuidanceTexts:
"""Mode-specific steering messages (verbatim from the legacy loops)."""
success: Callable[[int], str]
error: str
no_tools: Callable[[int], str]
def _standard_success_guidance(files_count: int) -> str:
return f"""✅ File implementation completed successfully!
📊 **Progress Status:** {files_count} files implemented
🎯 **Next Action:** Check if ALL files from the reproduction plan are implemented.
⚡ **Decision Process:**
1. **If ALL files implemented:** Reply with "All files implemented" to complete the task
2. **If MORE files need implementation:** Continue with dependency-aware workflow:
- **Use `write_file` to implement the new component"""
_STANDARD_ERROR_GUIDANCE = """❌ Error detected during file implementation.
🔧 **Action Required:**
1. Review the error details above
2. Fix the identified issue
3. **Check if ALL files from the reproduction plan are implemented:**
- **If YES:** Respond "**implementation complete**" to end the conversation
- **If NO:** Continue with proper development cycle for next file:
- **Use `write_file` to implement properly
4. Ensure proper error handling in future implementations"""
def _standard_no_tools_guidance(files_count: int) -> str:
return f"""⚠️ No tool calls detected in your response.
📊 **Current Progress:** {files_count} files implemented
🚨 **Action Required:** Check completion status NOW:
⚡ **Decision Process:**
1. **If ALL files from plan are implemented:** Reply "All files implemented" to complete
2. **If MORE files need implementation:** Use tools to continue:
- **Use `write_file` to implement the new component
🚨 **Critical:** Don't just explain - either declare completion or use tools!"""
def _indexed_success_guidance(files_count: int) -> str:
return f"""✅ File implementation completed successfully!
📊 **Progress Status:** {files_count} files implemented
🎯 **Next Action:** Check if ALL files from the reproduction plan are implemented.
⚡ **Decision Process:**
1. **If ALL files are implemented:** Use `execute_python` or `execute_bash` to test the complete implementation, then respond "**implementation complete**" to end the conversation
2. **If MORE files need implementation:** Continue with dependency-aware workflow:
- **Start with `read_code_mem`** to understand existing implementations and dependencies
- **Optionally use `search_code_references`** for reference patterns (OPTIONAL - use for inspiration only, original paper specs take priority)
- **Then `write_file`** to implement the new component
- **Finally: Test** if needed
💡 **Key Point:** Always verify completion status before continuing with new file creation."""
_INDEXED_ERROR_GUIDANCE = """❌ Error detected during file implementation.
🔧 **Action Required:**
1. Review the error details above
2. Fix the identified issue
3. **Check if ALL files from the reproduction plan are implemented:**
- **If YES:** Use `execute_python` or `execute_bash` to test the complete implementation, then respond "**implementation complete**" to end the conversation
- **If NO:** Continue with proper development cycle for next file:
- **Start with `read_code_mem`** to understand existing implementations
- **Use `write_file` to implement properly
4. Ensure proper error handling in future implementations"""
def _indexed_no_tools_guidance(files_count: int) -> str:
return f"""⚠️ No tool calls detected in your response.
📊 **Current Progress:** {files_count} files implemented
🚨 **Action Required:** Check completion status NOW:
⚡ **Decision Process:**
1. **If ALL files from plan are implemented:** Use `execute_python` or `execute_bash` to test, then reply "**implementation complete**"
2. **If MORE files need implementation:** Use tools to continue:
- **Start with `read_code_mem`** to understand existing implementations
- **Use `write_file` to implement the new component
🚨 **Critical:** Don't just explain - either declare completion or use tools!"""
_STANDARD_GUIDANCE = _GuidanceTexts(
success=_standard_success_guidance,
error=_STANDARD_ERROR_GUIDANCE,
no_tools=_standard_no_tools_guidance,
)
_INDEXED_GUIDANCE = _GuidanceTexts(
success=_indexed_success_guidance,
error=_INDEXED_ERROR_GUIDANCE,
no_tools=_indexed_no_tools_guidance,
)
class _InstrumentedTool(AliasedTool):
"""Model-facing tool that routes execution through the legacy tracker.
Execution is delegated to ``CodeImplementationAgent.execute_tool_calls``
so every domain behavior survives unchanged: read-tools-disabled mocks,
read_file → read_code_mem interception, write_file tracking + code
summaries. On top of that this wrapper applies the workflow-level
guards (loop detector) and bookkeeping (memory recording, progress)
that used to live inside the hand-rolled loop.
"""
def __init__(self, inner, alias: str, state: _RunState):
super().__init__(inner, alias)
self._state = state
async def execute(self, **kwargs: Any) -> Any:
state = self._state
if state.abort_reason:
return f"Error: tool execution aborted: {state.abort_reason}"
loop_status = state.loop_detector.check_tool_call(self.name)
if loop_status["should_stop"]:
state.abort_reason = loop_status["message"]
state.run_status = (
"aborted",
f"loop_detector tool-check: {loop_status['message']}",
)
state.logger.error(f"🛑 Tool execution aborted: {loop_status['message']}")
return f"Error: tool execution aborted: {loop_status['message']}"
legacy_call = {
"id": f"call_{uuid.uuid4().hex[:12]}",
"name": self.name,
"input": kwargs,
}
results = await state.code_tracker.execute_tool_calls([legacy_call])
result = results[0]["result"] if results else ""
# The legacy loop recorded success unconditionally (its ``isError``
# flag was never populated); error handling happens via guidance.
state.loop_detector.record_success()
state.memory_agent.record_tool_result(
tool_name=self.name, tool_input=kwargs, tool_result=result
)
if self.name == "write_file":
filename = kwargs.get("file_path", "unknown")
completed_first_time = state.progress_tracker.complete_file(
state.memory_agent.normalize_file_path(filename)
)
if completed_first_time:
print(f"✅ File completed: {filename}")
info = state.progress_tracker.get_progress_info()
state.emit_progress(
"Code implementation progress: "
f"{info['files_completed']}/{info['total_files']} files completed"
)
return result
class _ImplementationHook(AgentHook):
"""Per-iteration domain policy: guidance, memory strategy, round sync."""
def __init__(self, state: _RunState):
super().__init__()
self._state = state
async def before_execute_tools(self, context: AgentHookContext) -> None:
self._state.in_tools_phase = True
def finalize_content(self, context: AgentHookContext, content):
# Runs synchronously right after each model response — before the
# kernel's "after final response" injection drain — so the
# injection callback can already see an error finish_reason and
# refuse to keep the run alive after a provider failure.
self._state.last_finish_reason = getattr(
context.response, "finish_reason", None
)
return content
async def after_iteration(self, context: AgentHookContext) -> None:
state = self._state
try:
self._after_iteration(context)
except Exception: # noqa: BLE001 - policy must not kill the kernel loop
state.logger.exception("Implementation hook failed; continuing run")
finally:
state.in_tools_phase = False
state.iterations_done += 1
def _after_iteration(self, context: AgentHookContext) -> None:
state = self._state
response = context.response
state.last_finish_reason = getattr(response, "finish_reason", None)
files_count = state.code_tracker.get_files_implemented_count()
# Steering pressure after every tool round (tool outputs already
# flow back as protocol tool messages; only the guidance is added).
if context.tool_calls:
has_error = _tool_results_contain_error(context.tool_results)
guidance = (
state.guidance.error
if has_error
else state.guidance.success(files_count)
)
context.messages.append({"role": "user", "content": guidance})
# Sync file implementations into the memory agent.
summary = state.code_tracker.get_implementation_summary()
for file_info in summary.get("completed_files", []):
state.memory_agent.record_file_implementation(file_info["file"])
# Clean-slate memory strategy (unchanged ConciseMemoryAgent logic).
if state.memory_agent.should_trigger_memory_optimization(
context.messages, files_count
):
self._apply_memory_optimization(context, files_count)
state.memory_agent.start_new_round(iteration=state.iterations_done + 1)
# Emergency trim mirrors the legacy loop (a no-op unless a
# write_file optimization is pending — preserved behavior).
if len(context.messages) > _EMERGENCY_TRIM_THRESHOLD:
state.logger.warning(
"Emergency message trim - applying concise memory optimization"
)
self._apply_memory_optimization(context, files_count)
# File-level progress heartbeat.
info = state.progress_tracker.get_progress_info()
if info["total_files"] > 0 or info["files_completed"] > 0:
print(
f"📁 Files: {info['files_completed']}/{info['total_files']} "
f"({info['file_progress']:.1f}%)"
)
if info["estimated_remaining_seconds"] > 0:
print(
f"⏱️ Estimated remaining: {info['estimated_remaining_seconds']:.0f}s"
)
def _apply_memory_optimization(
self, context: AgentHookContext, files_count: int
) -> None:
state = self._state
system_messages = [
msg for msg in context.messages if msg.get("role") == "system"
]
non_system = [msg for msg in context.messages if msg.get("role") != "system"]
optimized = state.memory_agent.apply_memory_optimization(
state.system_prompt, non_system, files_count
)
if optimized is not non_system:
context.messages[:] = system_messages + list(optimized)
def _tool_results_contain_error(tool_results: List[Any]) -> bool:
"""Port of the legacy error sniffing over raw tool result strings."""
for result in tool_results:
text = result if isinstance(result, str) else str(result)
try:
parsed = json.loads(text)
if isinstance(parsed, dict) and parsed.get("status") == "error":
return True
continue
except (json.JSONDecodeError, TypeError):
pass
if "error" in text.lower():
return True
return False
class CodeImplementationWorkflow:
"""
Paper Code Implementation Workflow Manager (kernel-unified).
One class serves both modes:
- ``enable_indexing=False`` — the standard workflow (full tool surface,
general system prompt).
- ``enable_indexing=True`` — the indexed workflow (minimal
write_file/search_code_references surface, indexed system prompt).
"""
def __init__(
self,
enable_indexing: bool = False,
*,
require_verification: bool = False,
) -> None:
self.enable_indexing = enable_indexing
self.require_verification = require_verification
self.default_models = get_default_models()
self.logger = self._create_logger()
self.mcp_agent = None
self.enable_read_tools = True
self.loop_detector = LoopDetector()
self.progress_tracker = ProgressTracker()
self._last_run_state: Dict[str, Any] = {
"status": "unknown",
"reason": None,
"iterations": 0,
"elapsed_seconds": 0.0,
"files_completed": 0,
"total_files": 0,
"unimplemented_files": [],
}
# ==================== infrastructure ====================
def _create_logger(self) -> logging.Logger:
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
return logger
def _read_plan_file(self, plan_file_path: str) -> str:
plan_path = os.path.abspath(plan_file_path)
if not os.path.exists(plan_path):
raise FileNotFoundError(
f"Implementation plan file not found: {plan_file_path}"
)
with open(plan_path, "r", encoding="utf-8") as f:
return f.read()
def _check_file_tree_exists(self, target_directory: str) -> bool:
code_directory = os.path.join(target_directory, "generate_code")
return os.path.exists(code_directory) and len(os.listdir(code_directory)) > 0
@property
def _mcp_architecture(self) -> str:
return "indexed" if self.enable_indexing else "standard"
# ==================== public API ====================
async def run_workflow(
self,
plan_file_path: str,
target_directory: Optional[str] = None,
pure_code_mode: bool = False,
enable_read_tools: bool = True,
progress_callback: Optional[Callable] = None,
):
"""Run complete workflow - Main public interface."""
self.enable_read_tools = enable_read_tools
try:
plan_content = self._read_plan_file(plan_file_path)
if target_directory is None:
target_directory = str(os.path.dirname(os.path.abspath(plan_file_path)))
code_directory = os.path.join(target_directory, "generate_code")
self.logger.info("=" * 80)
self.logger.info("🚀 STARTING CODE IMPLEMENTATION WORKFLOW (kernel)")
self.logger.info("=" * 80)
self.logger.info(f"📄 Plan file: {plan_file_path}")
self.logger.info(f"📂 Plan file parent: {target_directory}")
self.logger.info(f"🎯 Code directory (MCP workspace): {code_directory}")
self.logger.info(f"🧭 Mode: {self._mcp_architecture}")
self.logger.info(
f"⚙️ Read tools: {'ENABLED' if self.enable_read_tools else 'DISABLED'}"
)
self.logger.info("=" * 80)
results = {}
if self._check_file_tree_exists(target_directory):
self.logger.info("File tree exists, skipping creation")
results["file_tree"] = "Already exists, skipped creation"
else:
self.logger.info("Creating file tree...")
results["file_tree"] = await self.create_file_structure(
plan_content, target_directory
)
if pure_code_mode:
self.logger.info("Starting pure code implementation...")
results["code_implementation"] = await self.implement_code_pure(
plan_content,
target_directory,
code_directory,
progress_callback=progress_callback,
)
run_state = dict(self._last_run_state)
generation_status = run_state.get("status", "unknown")
verification: list[dict[str, Any]] = []
if generation_status == "completed":
if self.require_verification:
verification = await self._verify_generated_code(
Path(code_directory)
)
tests_passed = bool(verification) and all(
result["passed"] for result in verification
)
if tests_passed:
self.logger.info(
"Workflow execution verified (all discovered tests passed)"
)
inner_status = "completed"
abort_reason = None
top_status = "success"
else:
inner_status = (
"unverified" if not verification else "test_failed"
)
abort_reason = (
"no_tests_discovered"
if not verification
else "generated_tests_failed"
)
self.logger.warning(
"Implementation generated but not verified: %s",
abort_reason,
)
top_status = "incomplete"
else:
inner_status = "completed"
abort_reason = run_state.get("reason")
top_status = "success"
else:
inner_status = generation_status
abort_reason = run_state.get("reason")
pending = run_state.get("unimplemented_files", []) or []
self.logger.warning(
"Workflow execution finished EARLY: status=%s reason=%s "
"(files=%d, %d unimplemented)",
inner_status,
run_state.get("reason"),
run_state.get("files_completed", 0),
len(pending),
)
if pending:
sample = ", ".join(pending[:5])
if len(pending) > 5:
sample += f", ... (+{len(pending) - 5} more)"
self.logger.warning("Unimplemented files: %s", sample)
top_status = "incomplete"
return {
"status": top_status,
"inner_status": inner_status,
"generation_status": generation_status,
"abort_reason": abort_reason,
"files_completed": run_state.get("files_completed", 0),
"total_files": run_state.get("total_files", 0),
"unimplemented_files": run_state.get("unimplemented_files", []),
"iterations": run_state.get("iterations", 0),
"elapsed_seconds": run_state.get("elapsed_seconds", 0.0),
"plan_file": plan_file_path,
"target_directory": target_directory,
"code_directory": os.path.join(target_directory, "generate_code"),
"results": results,
"verification": verification,
"mcp_architecture": self._mcp_architecture,
}
except Exception as e:
self.logger.error(f"Workflow execution failed: {e}")
code_directory = os.path.join(
target_directory
or str(os.path.dirname(os.path.abspath(plan_file_path))),
"generate_code",
)
return {
"status": "error",
"inner_status": "error",
"abort_reason": str(e),
"message": str(e),
"files_completed": 0,
"total_files": 0,
"unimplemented_files": [],
"elapsed_seconds": 0.0,
"plan_file": plan_file_path,
"target_directory": target_directory,
"code_directory": code_directory,
"results": {},
"mcp_architecture": self._mcp_architecture,
}
finally:
await self._cleanup_mcp_agent()
async def _verify_generated_code(
self, code_directory: Path
) -> list[dict[str, Any]]:
"""Run only mechanically discovered test commands, with bounded output."""
commands = discover_verification_commands(code_directory)
results: list[dict[str, Any]] = []
for command in commands:
try:
result = await asyncio.to_thread(
run_verification,
code_directory,
command,
timeout_seconds=300,
)
results.append(
{
"command_id": command.id,
"command": list(command.argv),
"passed": result.passed,
"exit_code": result.exit_code,
"timed_out": result.timed_out,
"duration_ms": result.duration_ms,
"stdout": result.stdout,
"stderr": result.stderr,
"output_truncated": result.output_truncated,
}
)
except OSError as exc:
results.append(
{
"command_id": command.id,
"command": list(command.argv),
"passed": False,
"error": f"{type(exc).__name__}: {exc}",
}
)
return results
async def create_file_structure(
self, plan_content: str, target_directory: str
) -> str:
"""Create file tree structure based on implementation plan."""
self.logger.info("Starting file tree creation...")
structure_agent = Agent(
name="StructureGeneratorAgent",
instruction=STRUCTURE_GENERATOR_PROMPT,
server_names=["command-executor"],
)
async with structure_agent:
creator = await attach_workflow_llm(
structure_agent,
phase="implementation",
)
message = f"""Analyze the following implementation plan and generate shell commands to create the file tree structure.
Target Directory: {target_directory}/generate_code/
Implementation Plan:
{plan_content}
Tasks:
1. Find the file tree structure in the implementation plan
2. Generate shell commands (mkdir -p, touch) to create that structure
3. Use the execute_commands tool to run the commands and create the file structure
Requirements:
- Use mkdir -p to create directories
- Use touch to create files
- Include __init__.py file for Python packages
- Use relative paths to the target directory
- Execute commands to actually create the file structure"""
result = await creator.generate_str(message=message)
self.logger.info(f"LLM response: {result[:200]}...")
code_dir = os.path.join(target_directory, "generate_code")
if not os.path.exists(code_dir):
self.logger.warning(
"LLM did not create directory, creating manually..."
)
os.makedirs(code_dir, exist_ok=True)
self.logger.info(f"✅ Manually created directory: {code_dir}")
else:
self.logger.info(f"✅ Directory exists: {code_dir}")
return result
async def implement_code_pure(
self,
plan_content: str,
target_directory: str,
code_directory: str = None,
progress_callback: Optional[Callable] = None,
) -> str:
"""Pure code implementation on the unified kernel."""
self.logger.info("Starting pure code implementation (no testing)...")
if code_directory is None:
code_directory = os.path.join(target_directory, "generate_code")
self.logger.info(f"🎯 Using code directory (MCP workspace): {code_directory}")
if not os.path.exists(code_directory):
self.logger.warning(
f"Code directory does not exist, creating it: {code_directory}"
)
os.makedirs(code_directory, exist_ok=True)
try:
provider, profile = get_workflow_provider(phase="implementation")
self.logger.info(
"Using DeepCode provider runtime: phase=%s provider=%s model=%s",
profile.phase,
profile.provider_name,
profile.model,
)
await self._initialize_mcp_agent(code_directory)
return await self._run_kernel_implementation(
provider,
plan_content,
target_directory,
code_directory,
progress_callback=progress_callback,
)
finally:
await self._cleanup_mcp_agent()
# ==================== kernel wiring ====================
def _system_prompt(self) -> str:
if self.enable_indexing:
return PURE_CODE_IMPLEMENTATION_SYSTEM_PROMPT_INDEX
return GENERAL_CODE_IMPLEMENTATION_SYSTEM_PROMPT
def _model_tool_names(self) -> List[str]:
names = list(
_INDEXED_TOOL_NAMES if self.enable_indexing else _STANDARD_TOOL_NAMES
)
if not self.enable_read_tools:
names = [n for n in names if n not in _READ_TOOL_NAMES]
return names
def _build_model_registry(self, state: _RunState) -> ToolRegistry:
"""Select + alias + instrument the model-facing tool surface."""
source = self.mcp_agent.tool_registry
aliased, missing = build_aliased_registry(source, self._model_tool_names())
if missing:
self.logger.warning(
"MCP tools missing from registry (server down?): %s", missing
)
instrumented = ToolRegistry()
for name in aliased.tool_names:
tool = aliased.get(name)
inner = tool.inner if isinstance(tool, AliasedTool) else tool
instrumented.register(_InstrumentedTool(inner, name, state))
self.logger.info(
"🔧 Model tool surface (%s): %s",
self._mcp_architecture,
instrumented.tool_names,
)
return instrumented
async def _run_kernel_implementation(
self,
provider,
plan_content: str,
target_directory: str,
code_directory: str,
progress_callback: Optional[Callable] = None,
) -> str:
system_prompt = self._system_prompt()
code_tracker = CodeImplementationAgent(
self.mcp_agent, self.logger, self.enable_read_tools
)
memory_agent = ConciseMemoryAgent(
plan_content,
self.logger,
target_directory,
self.default_models,
code_directory,
)
code_tracker.set_memory_agent(memory_agent, provider, "provider")
memory_agent.start_new_round(iteration=0)
total_files = len(memory_agent.all_files_list)
self.progress_tracker.set_total_files(total_files)
if progress_callback:
progress_callback(
85,
f"Code implementation started: 0/{total_files} planned files completed",
)
read_tools_status = "ENABLED" if self.enable_read_tools else "DISABLED"
self.logger.info(
f"🔧 Read tools (read_file, read_code_mem): {read_tools_status}"
)
state = _RunState(
memory_agent=memory_agent,
code_tracker=code_tracker,
loop_detector=self.loop_detector,
progress_tracker=self.progress_tracker,
logger=self.logger,
system_prompt=system_prompt,
guidance=_INDEXED_GUIDANCE if self.enable_indexing else _STANDARD_GUIDANCE,
progress_callback=progress_callback,
max_wall_seconds=_MAX_WALL_SECONDS,
)
implementation_message = f"""**Task: Implement code based on the following reproduction plan**
**Code Reproduction Plan:**
{plan_content}
**Working Directory:** {code_directory}
**Current Objective:** Begin implementation by analyzing the plan structure, examining the current project layout, and implementing the first foundation file according to the plan's priority order."""
initial_messages = [
{"role": "system", "content": system_prompt},
{"role": "user", "content": implementation_message},
]
async def should_stop() -> Optional[str]:
if state.run_status is not None:
return state.run_status[1]
if state.abort_reason:
state.run_status = (
"aborted",
f"loop_detector tool-check: {state.abort_reason}",
)
return state.abort_reason
elapsed = time.time() - state.start_time
if elapsed > state.max_wall_seconds:
reason = (
f"wall-clock budget exhausted after {elapsed:.0f}s "
f"(limit {int(state.max_wall_seconds)}s)"
)
state.run_status = ("max_time", reason)
self.logger.warning(f"Time limit reached: {elapsed:.2f}s")
return reason
if self.loop_detector.should_abort():
abort_reason = self.loop_detector.get_abort_reason()
state.run_status = (
"aborted",
f"loop_detector pre-LLM: {abort_reason}",
)
self.logger.error(f"🛑 Process aborted (pre-LLM): {abort_reason}")
return abort_reason or "loop detector abort"
if state.iterations_done >= 1:
unimplemented = memory_agent.get_unimplemented_files()
if not unimplemented:
state.run_status = (
"completed",
"all planned files implemented",
)
self.logger.info(
"✅ Code implementation complete - All files implemented"
)
return "all planned files implemented"
return None
async def inject_followups() -> List[Dict[str, Any]]:
# Only steer after a toolless final response; post-tool guidance
# is appended by the hook, and errors must end the run.
if state.in_tools_phase or state.run_status is not None:
return []
if state.last_finish_reason == "error":
return []
if not memory_agent.get_unimplemented_files():
return []
files_count = code_tracker.get_files_implemented_count()
return [{"role": "user", "content": state.guidance.no_tools(files_count)}]
async def on_retry_wait(message: str) -> None:
self.logger.warning("Implementation LLM retry: %s", message)
state.emit_progress(f"Retrying implementation LLM call: {message}")
# Security base. Legacy CLI/batch runs default to FULL_AUTO; an
# explicit ``default`` or ``plan`` setting tightens that behavior.
# The non-overridable sensitive-path denylist always applies (the
# agent can never read/write credential stores even if a plan asks).
#
# Mode + rules come from the ``security`` config block, overridable by
# DEEPCODE_PERMISSION_MODE. Any non-full_auto mode attaches a terminal
# approver so an `ask` becomes an interactive confirmation. An unknown
# value falls back to this legacy client's FULL_AUTO default.
security_cfg = getattr(get_runtime().config, "security", None)
permission_engine = build_permission_engine(security_cfg, cwd=code_directory)
mode = permission_engine.mode
approval_cb = None
if mode is not PermissionMode.FULL_AUTO:
approval_cb = TerminalApprover().as_async()
self.logger.info(
"🔐 Permission mode: %s (interactive approval)", mode.value
)
spec = AgentRunSpec(
initial_messages=initial_messages,
tools=self._build_model_registry(state),
model=provider.get_default_model(),
max_iterations=_MAX_ITERATIONS,
max_tool_result_chars=_MAX_TOOL_RESULT_CHARS,
temperature=0.2,
max_tokens=8192,
hook=_ImplementationHook(state),
session_key=f"code-implementation[{self._mcp_architecture}]",
provider_retry_mode="standard",
retry_wait_callback=on_retry_wait,
injection_callback=inject_followups,
should_stop_callback=should_stop,
permission_checker=permission_engine.evaluate,
approval_callback=approval_cb,
# The implementation phase owns its own budgets (wall clock +
# iteration caps); no per-call timeout, like the legacy loop.
llm_timeout_s=0,
)
runner = AgentRunner(provider)
result = await runner.run(spec)
status, reason = self._map_run_status(result, state, memory_agent)
elapsed_total = time.time() - state.start_time
self._last_run_state = {
"status": status,
"reason": reason,
"iterations": state.iterations_done,
"elapsed_seconds": elapsed_total,
"files_completed": len(memory_agent.get_implemented_files()),
"total_files": len(memory_agent.get_all_files_list()),
"unimplemented_files": list(memory_agent.get_unimplemented_files() or []),
}
return await self._generate_final_report(
state.iterations_done, elapsed_total, code_tracker, memory_agent
)
def _map_run_status(
self,
result,
state: _RunState,
memory_agent: ConciseMemoryAgent,
) -> tuple:
if state.run_status is not None:
return state.run_status
if result.stop_reason == "max_iterations":
return (
"max_iterations",
f"reached max_iterations={_MAX_ITERATIONS} without completion",
)
if result.stop_reason in ("error", "tool_error", "empty_final_response"):
return (
"incomplete",
f"LLM request failed during implementation: {result.error}",
)
if not memory_agent.get_unimplemented_files():
return ("completed", "all planned files implemented")
return (
"incomplete",
"model stopped while planned files remain unimplemented",
)
# ==================== MCP lifecycle ====================
async def _initialize_mcp_agent(self, code_directory: str):
"""Connect the MCP servers and set the workspace."""
try:
self.mcp_agent = Agent(
name="CodeImplementationAgent",
instruction=(
"You are a code implementation assistant, using MCP tools to "
"implement paper code replication. For large documents, use "
"document-segmentation tools to read content in smaller chunks "
"to avoid token limits."
),
server_names=list(_MCP_SERVER_NAMES),
)
await self.mcp_agent.__aenter__()
workspace_result = await self.mcp_agent.call_tool(
"set_workspace", {"workspace_path": code_directory}
)
self.logger.info(f"Workspace setup result: {workspace_result}")
except Exception as e: