error_learner.py

#!/usr/bin/env python3
"""
Error Learning System - Learn from validation failures

This script:
1. Tracks all validation errors
2. Identifies error patterns
3. Suggests improvements
4. Tracks fixes over time
"""

import json
import re
from pathlib import Path
from datetime import datetime
from collections import defaultdict, Counter

class ErrorLearner:
    """Learn from validation failures and track improvements"""
    
    def __init__(self, project_root):
        self.project_root = Path(project_root)
        self.error_db_path = self.project_root / '.dev' / 'comparison' / 'error_database.json'
        self.updates_path = self.project_root / '.dev' / 'planning' / 'AUTO_UPDATES.md'
        
        # Ensure directories exist
        self.error_db_path.parent.mkdir(parents=True, exist_ok=True)
        self.updates_path.parent.mkdir(parents=True, exist_ok=True)
        
        # Initialize error database
        self.error_db = self.load_error_database()
    
    def load_error_database(self):
        """Load existing error database or create new one"""
        if self.error_db_path.exists():
            with open(self.error_db_path, 'r') as f:
                return json.load(f)
        else:
            return {
                'errors': [],
                'patterns': {},
                'updates': [],
                'last_updated': None
            }
    
    def save_error_database(self):
        """Save error database to file"""
        self.error_db['last_updated'] = datetime.now().isoformat()
        
        with open(self.error_db_path, 'w') as f:
            json.dump(self.error_db, f, indent=2)
        
        print(f"✅ Error database saved to {self.error_db_path}")
    
    def record_error(self, app_id, error_data):
        """
        Record a validation error
        
        Args:
            app_id: Application identifier
            error_data: Dictionary with error information
        """
        error_entry = {
            'app_id': app_id,
            'timestamp': datetime.now().isoformat(),
            'errors': error_data.get('errors', []),
            'warnings': error_data.get('warnings', [])
        }
        
        self.error_db['errors'].append(error_entry)
        
        # Update patterns
        for error in error_data.get('errors', []):
            pattern_id = self.identify_pattern(error)
            if pattern_id:
                if pattern_id not in self.error_db['patterns']:
                    self.error_db['patterns'][pattern_id] = {
                        'count': 0,
                        'first_seen': datetime.now().isoformat(),
                        'apps': [],
                        'fixed': False
                    }
                
                self.error_db['patterns'][pattern_id]['count'] += 1
                if app_id not in self.error_db['patterns'][pattern_id]['apps']:
                    self.error_db['patterns'][pattern_id]['apps'].append(app_id)
        
        self.save_error_database()
    
    def identify_pattern(self, error_message):
        """
        Identify error pattern from message
        
        Args:
            error_message: Error message string
            
        Returns:
            Pattern identifier or None
        """
        error_lower = error_message.lower()
        
        # Common patterns
        patterns = {
            'json_syntax': ['json', 'syntax', 'parse error'],
            'missing_file': ['not found', 'missing', 'does not exist'],
            'invalid_config': ['invalid', 'configuration', 'config'],
            'dependency_error': ['dependency', 'module not found', 'import error'],
            'permission_error': ['permission denied', 'access denied'],
            'version_mismatch': ['version', 'mismatch', 'incompatible'],
            'security_issue': ['security', 'vulnerability', 'hardcoded'],
            'test_failure': ['test failed', 'assertion', 'expected']
        }
        
        for pattern_id, keywords in patterns.items():
            if any(keyword in error_lower for keyword in keywords):
                return pattern_id
        
        return 'unknown_error'
    
    def identify_new_patterns(self):
        """Identify error patterns that haven't been addressed"""
        new_patterns = []
        
        for pattern_id, pattern_data in self.error_db['patterns'].items():
            if not pattern_data.get('fixed', False) and pattern_data['count'] >= 2:
                new_patterns.append({
                    'pattern_id': pattern_id,
                    'count': pattern_data['count'],
                    'apps': pattern_data['apps'],
                    'first_seen': pattern_data['first_seen']
                })
        
        return sorted(new_patterns, key=lambda x: x['count'], reverse=True)
    
    def create_update_document(self, patterns):
        """Create improvement suggestions document"""
        if not patterns:
            print("✅ No new patterns to address")
            return
        
        content = f"""# Automatic Improvement Suggestions

Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}

## Overview

Found {len(patterns)} error patterns that need attention.

## Error Patterns

"""
        
        for i, pattern in enumerate(patterns, 1):
            content += f"""
### {i}. {pattern['pattern_id'].replace('_', ' ').title()}

**Occurrences**: {pattern['count']}  
**Affected Apps**: {', '.join(pattern['apps'])}  
**First Seen**: {pattern['first_seen']}

**Recommendation**:
- Review the error pattern
- Update validation logic if needed
- Add preventive checks
- Document the solution

---
"""
        
        content += """
## Next Steps

1. Review each pattern above
2. Implement fixes or improvements
3. Mark patterns as fixed in error database
4. Re-run tests to verify improvements

## How to Mark as Fixed

Edit `.dev/comparison/error_database.json` and set `"fixed": true` for resolved patterns.
"""
        
        with open(self.updates_path, 'w') as f:
            f.write(content)
        
        print(f"✅ Suggestions saved to {self.updates_path}")
    
    def get_statistics(self):
        """Get error learning statistics"""
        total_errors = len(self.error_db['errors'])
        unique_patterns = len(self.error_db['patterns'])
        fixed_patterns = sum(1 for p in self.error_db['patterns'].values() if p.get('fixed', False))
        unfixed_patterns = unique_patterns - fixed_patterns
        
        # Most common patterns
        pattern_counts = [(pid, pdata['count']) 
                         for pid, pdata in self.error_db['patterns'].items()]
        most_common = sorted(pattern_counts, key=lambda x: x[1], reverse=True)[:10]
        
        return {
            'total_errors_recorded': total_errors,
            'unique_patterns': unique_patterns,
            'fixed_patterns': fixed_patterns,
            'unfixed_patterns': unfixed_patterns,
            'most_common_patterns': most_common
        }

def main():
    """Command-line interface"""
    import sys
    import argparse
    
    parser = argparse.ArgumentParser(description='Error Learning System')
    parser.add_argument('command', choices=['stats', 'suggest'], 
                       help='Command to run')
    parser.add_argument('--project-root', default='.',
                       help='Project root directory')
    
    args = parser.parse_args()
    
    learner = ErrorLearner(args.project_root)
    
    if args.command == 'stats':
        stats = learner.get_statistics()
        
        print("\n" + "="*80)
        print("📊 ERROR LEARNING STATISTICS")
        print("="*80 + "\n")
        print(f"Total errors recorded: {stats['total_errors_recorded']}")
        print(f"Unique patterns: {stats['unique_patterns']}")
        print(f"Fixed patterns: {stats['fixed_patterns']}")
        print(f"Unfixed patterns: {stats['unfixed_patterns']}")
        
        if stats['most_common_patterns']:
            print("\n📈 Most common patterns:")
            for pattern, count in stats['most_common_patterns']:
                status = "✅" if learner.error_db['patterns'][pattern].get('fixed') else "❌"
                print(f"  {status} {pattern}: {count} occurrences")
        
        if stats['unfixed_patterns'] > 0:
            print(f"\n💡 Run 'python3 error_learner.py suggest' to generate improvement suggestions")
    
    elif args.command == 'suggest':
        print("\n" + "="*80)
        print("🔍 Analyzing error patterns...")
        print("="*80 + "\n")
        
        new_patterns = learner.identify_new_patterns()
        learner.create_update_document(new_patterns)
        
        if new_patterns:
            print(f"\n✅ Generated suggestions for {len(new_patterns)} error patterns")
        else:
            print("\n✅ No new error patterns detected!")

if __name__ == '__main__':
    main()