Repository navigation
Expand file tree
/
Copy pathloop_detector.py
More file actions
253 lines (215 loc) · 9.24 KB
/
Copy pathloop_detector.py
File metadata and controls
253 lines (215 loc) · 9.24 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
"""
Loop Detection and Timeout Safeguards for Code Implementation Workflow
This module provides tools to detect infinite loops, timeouts, and progress stalls
in the code implementation process to prevent hanging processes.
"""
import time
from typing import List, Dict, Any, Optional
class LoopDetector:
"""
Detects infinite loops, timeouts, and progress stalls in workflow execution.
Features:
- Track tool call history to detect repeated patterns
- Monitor time per file/operation
- Detect progress stalls
- Force stop after consecutive errors
"""
def __init__(
self,
max_repeats: int = 5,
timeout_seconds: int = 600,
stall_threshold: int = 300,
max_errors: int = 10,
):
"""
Initialize loop detector.
Args:
max_repeats: Maximum consecutive calls to same tool before flagging
timeout_seconds: Maximum time per file/operation (10 minutes default)
stall_threshold: Maximum time without progress (5 minutes default).
Increased from the original 180s because slow LLM calls (large
contexts, transient retries, network blips) can routinely take
several minutes — counting that idle wait as a "stall" used to
kill mid-run pipelines.
max_errors: Maximum consecutive errors before force stop
"""
self.max_repeats = max_repeats
self.timeout_seconds = timeout_seconds
self.stall_threshold = stall_threshold
self.max_errors = max_errors
# Tracking state
self.tool_history: List[str] = []
self.start_time = time.time()
self.last_progress_time = time.time()
self.consecutive_errors = 0
self.current_file = None
self.file_start_time = None
# Wall-clock budget that *excludes* LLM-call time. ``note_llm_wait``
# adds the elapsed LLM seconds back to ``last_progress_time`` so the
# stall check only penalises true tool-side inactivity.
self._pending_llm_offset_s: float = 0.0
def start_file(self, filename: str):
"""Start tracking a new file."""
self.current_file = filename
self.file_start_time = time.time()
self.last_progress_time = time.time()
print(f"📁 Starting file: {filename}")
def check_tool_call(self, tool_name: str) -> Dict[str, Any]:
"""
Check if tool call indicates a loop or timeout.
Args:
tool_name: Name of the tool being called
Returns:
Dict with status and warnings
"""
current_time = time.time()
self.tool_history.append(tool_name)
# Keep only recent history (last 10 calls)
if len(self.tool_history) > 10:
self.tool_history = self.tool_history[-10:]
# Check for repeated tool calls
if len(self.tool_history) >= self.max_repeats:
recent_tools = self.tool_history[-self.max_repeats :]
if len(set(recent_tools)) == 1: # All same tool
return {
"status": "loop_detected",
"message": f"⚠️ Loop detected: {tool_name} called {self.max_repeats} times consecutively",
"should_stop": True,
}
# Check file timeout
if (
self.file_start_time
and (current_time - self.file_start_time) > self.timeout_seconds
):
return {
"status": "timeout",
"message": f"⏰ Timeout: File {self.current_file} processing exceeded {self.timeout_seconds}s",
"should_stop": True,
}
# Check progress stall
if (current_time - self.last_progress_time) > self.stall_threshold:
return {
"status": "stall",
"message": f"🐌 Progress stall: No progress for {self.stall_threshold}s",
"should_stop": True,
}
# Check consecutive errors
if self.consecutive_errors >= self.max_errors:
return {
"status": "max_errors",
"message": f"❌ Too many errors: {self.consecutive_errors} consecutive errors",
"should_stop": True,
}
return {"status": "ok", "message": "Processing normally", "should_stop": False}
def record_progress(self):
"""Record that progress has been made."""
self.last_progress_time = time.time()
self.consecutive_errors = 0 # Reset error counter on progress
self._pending_llm_offset_s = 0.0
def note_llm_wait(self, elapsed_seconds: float) -> None:
"""Exclude an LLM-call wait from the stall budget.
Call this with ``time.time() - llm_start`` after every LLM
request/response cycle (including retries). The detector forwards
``last_progress_time`` by that amount so a slow LLM round-trip is
not mistaken for a frozen workflow. Has no effect if the call
completed quickly.
"""
if elapsed_seconds <= 0:
return
self.last_progress_time += elapsed_seconds
def record_error(self, error_message: str):
"""Record an error occurred."""
self.consecutive_errors += 1
print(f"❌ Error #{self.consecutive_errors}: {error_message}")
def record_success(self):
"""Record a successful operation."""
self.consecutive_errors = 0
self.record_progress()
def get_status_summary(self) -> Dict[str, Any]:
"""Get current status summary."""
current_time = time.time()
file_elapsed = (
(current_time - self.file_start_time) if self.file_start_time else 0
)
total_elapsed = current_time - self.start_time
return {
"current_file": self.current_file,
"file_elapsed_seconds": file_elapsed,
"total_elapsed_seconds": total_elapsed,
"consecutive_errors": self.consecutive_errors,
"recent_tools": self.tool_history[-5:], # Last 5 tools
"time_since_last_progress": current_time - self.last_progress_time,
}
def should_abort(self) -> bool:
"""Check if process should be aborted."""
status = self.check_tool_call("") # Check without adding to history
return status["should_stop"]
def get_abort_reason(self) -> Optional[str]:
"""Get reason for abort if should abort."""
if self.should_abort():
status = self.check_tool_call("")
return status["message"]
return None
class ProgressTracker:
"""
Track progress through implementation phases and files.
"""
def __init__(self, total_files: int = 0):
self.total_files = total_files
self.completed_files = 0
self.completed_file_paths = set()
self.current_phase = "Initializing"
self.phase_progress = 0
self.start_time = time.time()
def set_phase(self, phase_name: str, progress_percent: int):
"""Set current phase and progress percentage."""
self.current_phase = phase_name
self.phase_progress = progress_percent
print(f"📊 Progress: {progress_percent}% - {phase_name}")
@staticmethod
def _normalize_file_path(filename: str) -> str:
"""Normalize file paths so repeated writes do not inflate progress."""
return str(filename or "").replace("\\", "/").strip().strip("/")
def set_total_files(self, total_files: int):
"""Set the real planned file count."""
self.total_files = max(0, int(total_files or 0))
def complete_file(self, filename: str) -> bool:
"""Record completion of a unique file.
Returns ``True`` when this is the first completed write for the file,
``False`` when the same file was already counted.
"""
normalized = self._normalize_file_path(filename)
if normalized and normalized in self.completed_file_paths:
print(
f"ℹ️ File already counted: {filename} "
f"({self.completed_files}/{self.total_files})"
)
return False
if normalized:
self.completed_file_paths.add(normalized)
self.completed_files += 1
print(
f"✅ Completed file {self.completed_files}/{self.total_files}: {filename}"
)
return True
def get_progress_info(self) -> Dict[str, Any]:
"""Get current progress information."""
elapsed = time.time() - self.start_time
# Estimate remaining time
if self.completed_files > 0 and self.total_files > 0:
avg_time_per_file = elapsed / self.completed_files
remaining_files = self.total_files - self.completed_files
estimated_remaining = avg_time_per_file * remaining_files
else:
estimated_remaining = 0
return {
"phase": self.current_phase,
"phase_progress": self.phase_progress,
"files_completed": self.completed_files,
"total_files": self.total_files,
"file_progress": (self.completed_files / self.total_files * 100)
if self.total_files > 0
else 0,
"elapsed_seconds": elapsed,
"estimated_remaining_seconds": estimated_remaining,
}