LHR Data Migration Rake Tasks

# lib/tasks/lhr_migration.rake
namespace :lhr do
  desc "Complete migration of existing data to LHR FHIR structure"
  task migrate_all: :environment do |task, args|
    puts "🚀 Starting complete LHR migration..."
    start_time = Time.current
    
    # Run all migration subtasks
    Rake::Task['lhr:migrate_patient_problems'].invoke
    Rake::Task['lhr:migrate_clinical_impressions'].invoke
    Rake::Task['lhr:migrate_clinical_observations'].invoke  
    Rake::Task['lhr:migrate_diagnostic_reports'].invoke
    Rake::Task['lhr:backfill_fhir_resources'].invoke
    Rake::Task['lhr:create_initial_care_plans'].invoke
    Rake::Task['lhr:validate_migration'].invoke
    
    duration = Time.current - start_time
    puts "✅ LHR migration completed in #{duration.round(2)} seconds!"
  end

  desc "Migrate patient problems from historical prescription diagnoses"
  task migrate_patient_problems: :environment do
    puts "📋 Migrating patient problems from prescription diagnoses..."
    
    problems_created = 0
    problems_updated = 0
    
    # Process each clinic separately for performance
    Clinic.where(deleted: false).find_each do |clinic|
      puts "  Processing clinic: #{clinic.name} (ID: #{clinic.id})"
      
      # Get all patients for this clinic with prescriptions
      clinic_patients = ClinicPatient.includes(
        :patient, 
        prescriptions: [:prescription_complaints, :doctor]
      ).where(clinic_id: clinic.id)
      
      clinic_patients.find_each do |clinic_patient|
        patient = clinic_patient.patient
        problems_map = {}
        
        # Process prescriptions chronologically
        prescriptions = clinic_patient.prescriptions
                                     .where(is_final: true)
                                     .where.not(diagnosis: [nil, []])
                                     .order(:created_at)
        
        prescriptions.each do |prescription|
          prescription.diagnosis.each do |diagnosis|
            diagnosis_normalized = diagnosis.strip.downcase
            
            if problems_map[diagnosis_normalized]
              # Update existing problem with latest encounter
              problems_map[diagnosis_normalized][:last_encounter_id] = prescription.event_id
              problems_map[diagnosis_normalized][:clinical_notes] += "\\n#{prescription.created_at.strftime('%Y-%m-%d')}: #{diagnosis}"
            else
              # Create new problem entry
              complaint_text = prescription.prescription_complaints.pluck(:name).join(', ')
              
              problems_map[diagnosis_normalized] = {
                clinic_id: clinic.id,
                patient_id: patient.id,
                doctor_id: prescription.doctor_id,
                problem_code: extract_icd_code(diagnosis),
                problem_name: diagnosis,
                problem_category: categorize_diagnosis(diagnosis),
                severity: determine_severity(diagnosis),
                status: determine_status(diagnosis),
                onset_date: prescription.prescription_date || prescription.created_at.to_date,
                first_encounter_id: prescription.event_id,
                last_encounter_id: prescription.event_id,
                source_prescription_id: prescription.id,
                source_complaint_text: complaint_text,
                clinical_notes: "#{prescription.created_at.strftime('%Y-%m-%d')}: #{diagnosis}",
                metadata: {
                  migration_source: 'prescription_diagnosis',
                  original_prescription_id: prescription.id
                }
              }
            end
          end
        end
        
        # Create patient problems
        problems_map.values.each do |problem_data|
          begin
            existing_problem = PatientProblem.find_by(
              clinic_id: problem_data[:clinic_id],
              patient_id: problem_data[:patient_id],
              problem_name: problem_data[:problem_name]
            )
            
            if existing_problem
              existing_problem.update!(
                last_encounter_id: problem_data[:last_encounter_id],
                clinical_notes: problem_data[:clinical_notes]
              )
              problems_updated += 1
            else
              PatientProblem.create!(problem_data)
              problems_created += 1
            end
          rescue ActiveRecord::RecordInvalid => e
            puts "    ❌ Failed to create/update problem '#{problem_data[:problem_name]}' for patient #{patient.id}: #{e.message}"
          end
        end
      end
    end
    
    puts "  ✅ Created #{problems_created} new patient problems"
    puts "  ✅ Updated #{problems_updated} existing patient problems"
  end

  desc "Migrate clinical impressions from prescription diagnoses and observations"
  task migrate_clinical_impressions: :environment do
    puts "🩺 Migrating clinical impressions from prescriptions..."
    
    impressions_created = 0
    
    Prescription.where(is_final: true)
                .where('diagnosis IS NOT NULL OR observations IS NOT NULL')
                .where.not(diagnosis: [nil, []])
                .includes(:doctor)
                .find_each do |prescription|
      
      begin
        # Only create if we have meaningful clinical data
        next if prescription.diagnosis.blank? && prescription.observations.blank?
        
        summary_parts = []
        
        if prescription.prescription_complaints.any?
          complaints = prescription.prescription_complaints.pluck(:name).join(', ')
          summary_parts << "Chief Complaints: #{complaints}"
        end
        
        if prescription.observations.present?
          summary_parts << "Clinical Findings: #{prescription.observations}"
        end
        
        if prescription.diagnosis.any?
          summary_parts << "Assessment: #{prescription.diagnosis.join(', ')}"
        end
        
        # Extract ICD-10 codes
        finding_codes = prescription.diagnosis.map do |diagnosis|
          extract_icd_code(diagnosis)
        end.compact
        
        ClinicalImpression.create!(
          clinic_id: prescription.clinic_id,
          patient_id: prescription.patient_id,
          doctor_id: prescription.doctor_id,
          encounter_id: prescription.event_id,
          prescription_id: prescription.id,
          assessment_date: prescription.created_at,
          summary: summary_parts.join("\\n\\n"),
          finding_codes: finding_codes,
          finding_display: prescription.diagnosis || [],
          clinical_notes: prescription.observations,
          status: 'completed',
          metadata: {
            migration_source: 'prescription_diagnosis_observations',
            original_prescription_id: prescription.id
          }
        )
        impressions_created += 1
      rescue ActiveRecord::RecordInvalid => e
        puts "  ❌ Failed to create clinical impression for prescription #{prescription.id}: #{e.message}"
      end
    end
    
    puts "  ✅ Created #{impressions_created} clinical impressions"
  end

  desc "Migrate clinical observations from prescription observations and appointment vitals"
  task migrate_clinical_observations: :environment do
    puts "🔬 Migrating clinical observations..."
    
    observations_created = 0
    
    # Migrate from prescription observations
    Prescription.where(is_final: true)
                .where.not(observations: [nil, ''])
                .includes(:doctor)
                .find_each do |prescription|
      
      begin
        ClinicalObservation.create!(
          clinic_id: prescription.clinic_id,
          patient_id: prescription.patient_id,
          doctor_id: prescription.doctor_id,
          encounter_id: prescription.event_id,
          prescription_id: prescription.id,
          category: 'exam',
          code: 'CLINICAL-EXAM',
          display_name: 'Clinical Examination',
          value_type: 'string',
          value_string: prescription.observations,
          interpretation: 'normal', # Default, could be enhanced
          notes: 'Migrated from prescription observations',
          metadata: {
            migration_source: 'prescription_observations',
            original_prescription_id: prescription.id
          }
        )
        observations_created += 1
      rescue ActiveRecord::RecordInvalid => e
        puts "  ❌ Failed to create observation for prescription #{prescription.id}: #{e.message}"
      end
    end
    
    # Migrate from appointment vitals
    Appointment.where.not(vitals: [nil, '{}', ''])
               .includes(:clinic_doctor)
               .find_each do |appointment|
      
      begin
        vitals_data = appointment.vitals.is_a?(String) ? JSON.parse(appointment.vitals) : appointment.vitals
        doctor_id = appointment.clinic_doctor&.doctor_id
        
        vitals_data.each do |vital_key, vital_value|
          next unless vital_value.present?
          
          standardized_key = standardize_vital_key(vital_key)
          next unless standardized_key
          
          # Parse value and determine type
          parsed_value = parse_vital_value_for_observation(vital_key, vital_value)
          
          ClinicalObservation.create!(
            clinic_id: appointment.clinic_id,
            patient_id: appointment.clinic_patient.patient_id,
            doctor_id: doctor_id,
            encounter_id: appointment.id,
            category: 'vital-signs',
            code: get_loinc_code(standardized_key),
            display_name: get_vital_display_name(standardized_key),
            value_type: parsed_value[:type],
            value_quantity: parsed_value[:quantity],
            value_string: parsed_value[:string],
            value_unit: get_vital_unit(standardized_key),
            reference_range: get_normal_range(standardized_key),
            interpretation: is_vital_abnormal?(vital_key, vital_value) ? 'abnormal' : 'normal',
            metadata: {
              migration_source: 'appointment_vitals',
              original_appointment_id: appointment.id,
              original_vital_key: vital_key
            }
          )
          observations_created += 1
        end
      rescue JSON::ParserError, ActiveRecord::RecordInvalid => e
        puts "  ❌ Failed to process vitals for appointment #{appointment.id}: #{e.message}"
      end
    end
    
    puts "  ✅ Created #{observations_created} clinical observations"
  end

  desc "Create diagnostic reports from prescription diagnoses"
  task migrate_diagnostic_reports: :environment do
    puts "📊 Creating diagnostic reports from prescription diagnoses..."
    
    reports_created = 0
    
    Prescription.where(is_final: true)
                .where.not(diagnosis: [nil, []])
                .includes(:doctor)
                .find_each do |prescription|
      
      begin
        DiagnosticReport.create!(
          clinic_id: prescription.clinic_id,
          patient_id: prescription.patient_id,
          doctor_id: prescription.doctor_id,
          encounter_id: prescription.event_id,
          prescription_id: prescription.id,
          category: 'general',
          code: 'CLINICAL-ASSESSMENT',
          display_name: 'Clinical Assessment',
          conclusion: prescription.diagnosis.join('; '),
          coded_diagnosis: prescription.diagnosis_codes || [],
          clinical_significance: determine_clinical_significance(prescription.diagnosis),
          clinical_notes: prescription.notes,
          status: 'final',
          metadata: {
            migration_source: 'prescription_diagnosis',
            original_prescription_id: prescription.id
          }
        )
        reports_created += 1
      rescue ActiveRecord::RecordInvalid => e
        puts "  ❌ Failed to create diagnostic report for prescription #{prescription.id}: #{e.message}"
      end
    end
    
    puts "  ✅ Created #{reports_created} diagnostic reports"
  end

  desc "Backfill FHIR resources for finalized prescriptions"
  task backfill_fhir_resources: :environment do
    puts "⚡ Backfilling FHIR resources for finalized prescriptions..."
    
    jobs_queued = 0
    
    # Only process prescriptions from last 6 months to avoid overwhelming the system
    Prescription.where(is_final: true)
                .where('created_at > ?', 6.months.ago)
                .find_each do |prescription|
      
      # Check if FHIR resources already exist
      has_medication_requests = MedicationRequest.exists?(prescription_id: prescription.id)
      has_service_requests = ServiceRequest.exists?(prescription_id: prescription.id)
      
      unless has_medication_requests && has_service_requests
        Prescriptions::CreateFhirResourcesJob.perform_later(prescription.id)
        jobs_queued += 1
      end
    end
    
    puts "  ✅ Queued #{jobs_queued} background jobs for FHIR resource creation"
  end

  desc "Create initial care plans for patients with chronic conditions"
  task create_initial_care_plans: :environment do
    puts "📋 Creating initial care plans for chronic conditions..."
    
    care_plans_created = 0
    
    # Find patients with chronic problems
    chronic_problems = PatientProblem.where(
      status: ['chronic', 'active'],
      severity: ['high', 'critical']
    ).group_by { |p| [p.clinic_id, p.patient_id] }
    
    chronic_problems.each do |(clinic_id, patient_id), problems|
      next if problems.empty?
      
      # Check if care plan already exists
      existing_plan = CarePlan.find_by(
        clinic_id: clinic_id,
        patient_id: patient_id,
        status: 'active'
      )
      next if existing_plan
      
      # Get primary doctor (most recent prescriptions)
      primary_doctor_id = Prescription.where(
        clinic_id: clinic_id,
        patient_id: patient_id
      ).order(created_at: :desc).first&.doctor_id
      
      next unless primary_doctor_id
      
      begin
        care_plan = CarePlan.create!(
          clinic_id: clinic_id,
          patient_id: patient_id,
          doctor_id: primary_doctor_id,
          status: 'active',
          intent: 'plan',
          title: 'Chronic Disease Management Plan',
          description: \"Care plan for managing #{problems.map(&:problem_name).join(', ')}\",
          period_start: Date.current,
          addresses_problems: problems.map(&:id),
          goals: {
            'primary' => 'Stabilize chronic conditions',
            'secondary' => 'Prevent complications',
            'tertiary' => 'Improve quality of life'
          },
          activities: {
            'monitoring' => 'Regular vital signs and lab monitoring',
            'medication' => 'Medication adherence and optimization',
            'lifestyle' => 'Diet and exercise counseling'
          },
          care_team: [
            {
              'role' => 'Primary Doctor',
              'name' => problems.first.doctor.first_name + ' ' + problems.first.doctor.last_name
            }
          ],
          metadata: {
            migration_source: 'chronic_problems_aggregation',
            problem_count: problems.length
          }
        )
        care_plans_created += 1
      rescue ActiveRecord::RecordInvalid => e
        puts "  ❌ Failed to create care plan for patient #{patient_id} in clinic #{clinic_id}: #{e.message}"
      end
    end
    
    puts "  ✅ Created #{care_plans_created} initial care plans"
  end

  desc "Validate migration data integrity"
  task validate_migration: :environment do
    puts "🔍 Validating migration data integrity..."
    
    # Check prescription -> FHIR resource mapping
    finalized_prescriptions = Prescription.where(is_final: true).count
    medication_requests = MedicationRequest.count
    service_requests = ServiceRequest.count
    
    puts "  📊 Migration Statistics:"
    puts "    Finalized Prescriptions: #{finalized_prescriptions}"
    puts "    Patient Problems: #{PatientProblem.count}"
    puts "    Clinical Impressions: #{ClinicalImpression.count}"
    puts "    Clinical Observations: #{ClinicalObservation.count}"
    puts "    Diagnostic Reports: #{DiagnosticReport.count}"
    puts "    Medication Requests: #{medication_requests}"
    puts "    Service Requests: #{service_requests}"
    puts "    Care Plans: #{CarePlan.count}"
    
    # Validate data consistency
    orphaned_clinical_impressions = ClinicalImpression.left_joins(:prescription).where(prescriptions: { id: nil }).count
    orphaned_medication_requests = MedicationRequest.left_joins(:prescription).where(prescriptions: { id: nil }).count
    orphaned_service_requests = ServiceRequest.left_joins(:prescription).where(prescriptions: { id: nil }).count
    orphaned_problems = PatientProblem.left_joins(:patient).where(patients: { id: nil }).count
    
    if orphaned_clinical_impressions > 0 || orphaned_medication_requests > 0 || orphaned_service_requests > 0 || orphaned_problems > 0
      puts "  ⚠️  Data integrity issues found:"
      puts "    Orphaned Clinical Impressions: #{orphaned_clinical_impressions}" if orphaned_clinical_impressions > 0
      puts "    Orphaned Medication Requests: #{orphaned_medication_requests}" if orphaned_medication_requests > 0
      puts "    Orphaned Service Requests: #{orphaned_service_requests}" if orphaned_service_requests > 0  
      puts "    Orphaned Patient Problems: #{orphaned_problems}" if orphaned_problems > 0
    else
      puts "  ✅ Data integrity validation passed"
    end
  end

  desc "Rollback migration (USE WITH CAUTION)"
  task rollback_migration: :environment do
    puts "🚨 Rolling back LHR migration..."
    
    print "Are you sure you want to delete all migrated LHR data? (yes/no): "
    confirmation = STDIN.gets.chomp
    
    if confirmation.downcase == 'yes'
      deleted_counts = {}
      
      deleted_counts['Medication Requests'] = MedicationRequest.where("metadata->>'migration_source' IS NOT NULL").delete_all
      deleted_counts['Service Requests'] = ServiceRequest.where("metadata->>'migration_source' IS NOT NULL").delete_all
      deleted_counts['Patient Problems'] = PatientProblem.where("metadata->>'migration_source' IS NOT NULL").delete_all
      deleted_counts['Clinical Impressions'] = ClinicalImpression.where("metadata->>'migration_source' IS NOT NULL").delete_all
      deleted_counts['Clinical Observations'] = ClinicalObservation.where("metadata->>'migration_source' IS NOT NULL").delete_all
      deleted_counts['Diagnostic Reports'] = DiagnosticReport.where("metadata->>'migration_source' IS NOT NULL").delete_all
      deleted_counts['Care Plans'] = CarePlan.where("metadata->>'migration_source' IS NOT NULL").delete_all
      
      puts "  ✅ Rollback completed:"
      deleted_counts.each { |type, count| puts "    #{type}: #{count} records deleted" }
    else
      puts "  Rollback cancelled"
    end
  end

  # Helper methods for the tasks
  def categorize_diagnosis(diagnosis)
    case diagnosis.downcase
    when /diabetes|glucose|insulin/ then 'Endocrine'
    when /hypertension|blood pressure|cardiac|heart/ then 'Cardiovascular'
    when /infection|fever|bacteria|virus/ then 'Infectious'
    when /pain|arthritis|joint|muscle/ then 'Musculoskeletal'
    when /respiratory|lung|asthma|copd/ then 'Respiratory'
    when /gastro|stomach|digestive/ then 'Gastrointestinal'
    when /mental|depression|anxiety/ then 'Mental Health'
    when /skin|dermat/ then 'Dermatological'
    else 'General'
    end
  end

  def determine_severity(diagnosis)
    case diagnosis.downcase
    when /cancer|tumor|malignant|stroke|heart attack|myocardial/ then 'critical'
    when /diabetes|hypertension|copd|asthma|chronic/ then 'high'
    when /infection|acute|severe/ then 'medium'
    else 'low'
    end
  end

  def determine_status(diagnosis)
    chronic_keywords = ['diabetes', 'hypertension', 'copd', 'asthma', 'arthritis', 'chronic']
    
    if chronic_keywords.any? { |keyword| diagnosis.downcase.include?(keyword) }
      'chronic'
    else
      'active'
    end
  end

  def extract_icd_code(diagnosis)
    # Simple ICD-10 code mapping (can be enhanced with proper medical coding)
    case diagnosis.downcase
    when /type.*2.*diabetes/ then 'E11.9'
    when /diabetes/ then 'E10.9'
    when /hypertension/ then 'I10'
    when /hyperlipidemia/ then 'E78.5'
    when /asthma/ then 'J45.9'
    when /copd/ then 'J44.1'
    else nil
    end
  end

  def standardize_vital_key(key)
    case key.to_s.downcase
    when 'bp', 'blood_pressure', 'bloodpressure' then 'bloodPressure'
    when 'hr', 'heart_rate', 'heartrate', 'pulse' then 'heartRate'
    when 'temp', 'temperature' then 'temperature'
    when 'weight', 'wt' then 'weight'
    when 'spo2', 'oxygen_saturation', 'o2sat' then 'oxygenSaturation'
    when 'rr', 'respiratory_rate', 'resp_rate' then 'respiratoryRate'
    when 'bmi' then 'bmi'
    else nil
    end
  end

  def get_loinc_code(vital_type)
    # Standard LOINC codes for vital signs
    {
      'bloodPressure' => '85354-9',
      'heartRate' => '8867-4',
      'temperature' => '8310-5',
      'weight' => '29463-7',
      'oxygenSaturation' => '2708-6',
      'respiratoryRate' => '9279-1',
      'bmi' => '39156-5'
    }[vital_type] || 'unknown'
  end

  def get_vital_display_name(vital_type)
    {
      'bloodPressure' => 'Blood Pressure',
      'heartRate' => 'Heart Rate',
      'temperature' => 'Body Temperature',
      'weight' => 'Weight',
      'oxygenSaturation' => 'Oxygen Saturation',
      'respiratoryRate' => 'Respiratory Rate',
      'bmi' => 'Body Mass Index'
    }[vital_type] || vital_type.humanize
  end

  def get_vital_unit(vital_type)
    {
      'bloodPressure' => 'mmHg',
      'heartRate' => 'bpm',
      'temperature' => '°C',
      'weight' => 'kg',
      'oxygenSaturation' => '%',
      'respiratoryRate' => 'breaths/min',
      'bmi' => 'kg/m²'
    }[vital_type] || ''
  end

  def parse_vital_value_for_observation(key, value)
    standardized_key = standardize_vital_key(key)
    
    if standardized_key == 'bloodPressure' && value.to_s.include?('/')
      # Store as string for blood pressure
      { type: 'string', string: value.to_s, quantity: nil }
    else
      # Store as quantity for numeric values
      numeric_value = value.to_s.gsub(/[^\d.]/, '').to_f
      { type: 'quantity', quantity: numeric_value, string: nil }
    end
  end

  def get_normal_range(vital_type)
    {
      'bloodPressure' => { 'low' => '90/60', 'high' => '140/90' },
      'heartRate' => { 'low' => 60, 'high' => 100 },
      'temperature' => { 'low' => 36.0, 'high' => 37.5 },
      'oxygenSaturation' => { 'low' => 95, 'high' => 100 },
      'respiratoryRate' => { 'low' => 12, 'high' => 20 }
    }[vital_type] || {}
  end

  def is_vital_abnormal?(key, value)
    standardized_key = standardize_vital_key(key)
    normal_ranges = get_normal_range(standardized_key)
    return false if normal_ranges.empty?

    if standardized_key == 'bloodPressure' && value.to_s.include?('/')
      systolic, diastolic = value.to_s.split('/').map(&:to_i)
      min_sys, min_dia = normal_ranges['low'].split('/').map(&:to_i)
      max_sys, max_dia = normal_ranges['high'].split('/').map(&:to_i)
      
      systolic < min_sys || systolic > max_sys || diastolic < min_dia || diastolic > max_dia
    else
      numeric_value = value.to_s.gsub(/[^\d.]/, '').to_f
      numeric_value < normal_ranges['low'] || numeric_value > normal_ranges['high']
    end
  end

  def determine_clinical_significance(diagnoses)
    critical_conditions = ['stroke', 'heart attack', 'cancer', 'myocardial infarction']
    abnormal_conditions = ['diabetes', 'hypertension', 'copd', 'asthma']
    
    diagnoses_text = diagnoses.join(' ').downcase
    
    if critical_conditions.any? { |condition| diagnoses_text.include?(condition) }
      'critical'
    elsif abnormal_conditions.any? { |condition| diagnoses_text.include?(condition) }
      'abnormal'
    else
      'normal'
    end
  end
end

# Run complete migration
cd backend
bundle exec rake lhr:migrate_all

# Or run individual steps
bundle exec rake lhr:migrate_patient_problems
bundle exec rake lhr:migrate_clinical_observations
bundle exec rake lhr:migrate_diagnostic_reports
bundle exec rake lhr:backfill_fhir_resources
bundle exec rake lhr:create_initial_care_plans

# Validate migration results
bundle exec rake lhr:validate_migration

# Backfill FHIR resources for new finalized prescriptions
bundle exec rake lhr:backfill_fhir_resources

# Create care plans for new chronic patients
bundle exec rake lhr:create_initial_care_plans

# Only if migration needs to be completely reversed
bundle exec rake lhr:rollback_migration