Integration Failures Troubleshooting

# Check MSG91 API connectivity
begin
  response = HTTParty.post(
    'https://api.msg91.com/api/v2/sendsms',
    body: test_payload.to_json,
    headers: { 'authkey' => Rails.application.credentials.msg91_auth_key },
    timeout: 10
  )
  puts "MSG91 Status: #{response.code}"
  puts "Response: #{response.body}"
rescue Net::TimeoutError => e
  puts "Timeout error: #{e.message}"
rescue => e
  puts "Connection error: #{e.message}"
end

# Check recent SMS failures
failed_sms = MsgLog.where(
  msg_type: 'sms',
  msg_status: 'FAILED'
).where('created_at > ?', 1.hour.ago)

puts "Failed SMS count: #{failed_sms.count}"
failed_sms.limit(5).each do |msg|
  puts "Failure reason: #{msg.failure_reason}"
end

# 1. Verify API credentials
creds = Rails.application.credentials.msg91_auth_key
puts "MSG91 Key present: #{creds.present?}"
puts "Key format valid: #{creds&.length == 24}"  # MSG91 keys are typically 24 chars

# 2. Implement retry logic with exponential backoff
class Sms::Base < ActiveJob::Base
  retry_on Net::TimeoutError, wait: :exponentially_longer, attempts: 3
  retry_on HTTParty::Error, wait: :exponentially_longer, attempts: 3
  
  def perform(phone_number, message_text, dlt_id = nil)
    # SMS sending logic with timeout handling
    begin
      response = HTTParty.post(endpoint, options.merge(timeout: 30))
    rescue Net::TimeoutError
      Rails.logger.error "MSG91 timeout for #{phone_number}"
      raise  # Will trigger retry
    end
  end
end

# 3. Check and increase timeout settings
HTTParty.post(
  endpoint, 
  options.merge(
    timeout: 30,        # Connection timeout
    read_timeout: 45    # Read timeout
  )
)

# Check DLT template usage
template_usage = MsgLog.where(msg_type: 'sms')
                       .where('created_at > ?', 1.day.ago)
                       .group(:unique_identifier)
                       .count

puts "Template usage:"
template_usage.each do |template_id, count|
  puts "Template #{template_id}: #{count} messages"
end

# Check specific template issues
failed_templates = MsgLog.where(
  msg_type: 'sms',
  msg_status: 'FAILED'
).where('failure_reason LIKE ?', '%template%')
.group(:unique_identifier)
.count

puts "Failed templates: #{failed_templates}"

# 1. Update DLT template mappings
DLT_TEMPLATES = {
  'appointment_confirmation' => '1707160734083863936',
  'verification_code' => '1707160734083863937',
  'appointment_reminder' => '1707160734083863938'
}.freeze

def get_dlt_template_id(message_type)
  template_id = DLT_TEMPLATES[message_type]
  Rails.logger.warn "Missing DLT template for: #{message_type}" if template_id.blank?
  template_id
end

# 2. Implement template validation
def validate_dlt_template(template_id, message_text)
  # Check if message matches approved template structure
  # Implement template validation logic
end

# 3. Add fallback for failed templates
if response.code == 400 && response.body.include?('template')
  # Try with generic template
  fallback_template_id = DLT_TEMPLATES['generic']
  retry_with_template(fallback_template_id)
end

# Check WhatsApp message tracking
whatsapp_messages = MsgLog.where(msg_type: 'whatsapp')
                          .where('created_at > ?', 1.hour.ago)

puts "Total WhatsApp messages: #{whatsapp_messages.count}"
puts "Delivered: #{whatsapp_messages.where(msg_status: 'DELIVERED').count}"
puts "Failed: #{whatsapp_messages.where(msg_status: 'FAILED').count}"
puts "Still pending: #{whatsapp_messages.where(msg_status: 'SENT').count}"

# Check webhook matching rate
total_webhooks = WatiWebhook.where('created_at > ?', 1.hour.ago).count
matched_webhooks = WatiWebhook.where('created_at > ?', 1.hour.ago)
                              .where('msg_log_id IS NOT NULL').count

puts "Webhook match rate: #{(matched_webhooks.to_f / total_webhooks * 100).round(2)}%" if total_webhooks > 0

# Check recent webhook errors
recent_webhooks = WatiWebhook.where('created_at > ?', 1.hour.ago)
                             .where('msg_log_id IS NULL')

recent_webhooks.limit(5).each do |webhook|
  puts "Unmatched webhook: #{webhook.payload['eventType']} - #{webhook.payload['whatsappMessageId']}"
end

# 1. Improve webhook matching logic
def find_matching_msg_log(webhook_data)
  # Direct matching for failed messages (most accurate)
  if webhook_data['eventType'] == 'templateMessageFailed' && webhook_data['localMessageId']
    return MsgLog.find_by(wati_local_message_id: webhook_data['localMessageId'])
  end
  
  # Direct matching using stored whatsapp message ID
  if webhook_data['whatsappMessageId'].present?
    msg_log = MsgLog.find_by(wati_message_id: webhook_data['whatsappMessageId'])
    return msg_log if msg_log
  end
  
  # Probabilistic matching with improved accuracy
  phone_number = extract_phone_from_webhook(webhook_data)
  template_name = webhook_data['templateName']
  webhook_time = Time.at(webhook_data['timestamp'].to_i)
  
  # Narrow time window for better accuracy
  candidates = MsgLog.where(
    phone_number: phone_number,
    msg_type: 'whatsapp',
    created_at: (webhook_time - 2.minutes)..(webhook_time + 2.minutes)
  )
  
  # Template-based matching
  if template_name.present?
    template_match = candidates.where(unique_identifier: template_name)
                              .where(wati_message_id: nil)  # Not already matched
                              .order(:created_at)
                              .first
    return template_match if template_match
  end
  
  # Fallback to closest time match
  candidates.where(wati_message_id: nil).order(:created_at).first
end

# 2. Add webhook retry mechanism
class WatiWebhooksController
  def create
    webhook = WatiWebhook.create!(webhook_params)
    
    # Process immediately
    result = process_delivery_webhook(webhook)
    
    # Schedule retry if matching failed
    if !result && webhook.delivery_status_event?
      RetryWebhookProcessingJob.set(wait: 5.minutes).perform_later(webhook.id)
    end
    
    head :ok
  end
end

# 3. Monitor and alert on low match rates
def webhook_health_alert
  recent_webhooks = WatiWebhook.where('created_at > ?', 1.hour.ago)
  match_rate = recent_webhooks.where('msg_log_id IS NOT NULL').count.to_f / recent_webhooks.count
  
  if match_rate < 0.8  # Alert if less than 80% match rate
    SlackAlerts.perform_later(
      "Low WhatsApp webhook match rate: #{(match_rate * 100).round(2)}%"
    )
  end
end

# Test webhook signature verification
webhook_body = request.raw_post
webhook_signature = request.headers['X-Razorpay-Signature']
webhook_secret = Rails.application.credentials.razorpay_webhook_secret

begin
  is_valid = Razorpay::Utility.verify_webhook_signature(
    webhook_body,
    webhook_signature, 
    webhook_secret
  )
  puts "Webhook signature valid: #{is_valid}"
rescue => e
  puts "Signature verification error: #{e.message}"
end

# Check recent webhook failures
failed_webhooks = RazorpayWebhookError.where('created_at > ?', 1.hour.ago)
puts "Failed webhook count: #{failed_webhooks.count}"
failed_webhooks.each do |error|
  puts "Error: #{error.error_message}"
end

# 1. Verify webhook secret configuration
webhook_secret = Rails.application.credentials.razorpay_webhook_secret
puts "Webhook secret configured: #{webhook_secret.present?}"

# In Razorpay dashboard, ensure webhook secret matches the configured value

# 2. Improve webhook signature handling
class BillingWebhooksController
  def webhooks
    webhook_body = request.raw_post
    webhook_signature = request.headers['X-Razorpay-Signature']
    
    # Verify signature
    begin
      unless Razorpay::Utility.verify_webhook_signature(
        webhook_body, 
        webhook_signature, 
        Rails.application.credentials.razorpay_webhook_secret
      )
        Rails.logger.error "Invalid Razorpay webhook signature"
        head :bad_request and return
      end
    rescue => e
      Rails.logger.error "Webhook signature error: #{e.message}"
      head :bad_request and return
    end
    
    # Process webhook
    webhook_data = JSON.parse(webhook_body)
    result = Webhooks::Receive.call(webhook_data: webhook_data)
    
    head result.success? ? :ok : :unprocessable_entity
  end
end

# 3. Add webhook retry mechanism
# Configure webhook endpoint in Razorpay with retry settings

# Check payment status
payment_id = "pay_abc123"
begin
  payment = Razorpay::Payment.fetch(payment_id)
  puts "Payment status: #{payment.status}"
  puts "Payment method: #{payment.method}"
  puts "Amount: #{payment.amount}"
  puts "Captured: #{payment.captured}"
rescue => e
  puts "Payment fetch error: #{e.message}"
end

# Check recent payment failures
failed_payments = BillingPayment.joins(:basic_billing_payment_status)
                                .where(basic_billing_payment_statuses: { default_name: 'failed' })
                                .where('billing_payments.created_at > ?', 1.hour.ago)

puts "Failed payments: #{failed_payments.count}"
failed_payments.each do |payment|
  puts "Payment ID: #{payment.razorpay_payment_id}, Reason: #{payment.failure_reason}"
end

# 1. Implement automatic payment capture
def capture_payment(payment_id, amount)
  begin
    payment = Razorpay::Payment.fetch(payment_id)
    
    if payment.status == 'authorized'
      captured_payment = payment.capture(amount: amount)
      puts "Payment captured: #{captured_payment.id}"
      return captured_payment
    else
      Rails.logger.warn "Payment #{payment_id} status is #{payment.status}, cannot capture"
    end
  rescue => e
    Rails.logger.error "Payment capture failed: #{e.message}"
    raise
  end
end

# 2. Add payment reconciliation job
class PaymentReconciliationJob < ApplicationJob
  def perform
    # Find authorized payments older than 1 hour
    old_payments = BillingPayment.where(razorpay_payment_id: present)
                                 .where('created_at < ?', 1.hour.ago)
    
    old_payments.each do |billing_payment|
      begin
        razorpay_payment = Razorpay::Payment.fetch(billing_payment.razorpay_payment_id)
        
        # Update local status based on Razorpay status
        if razorpay_payment.captured && billing_payment.status != 'captured'
          billing_payment.update!(status: 'captured')
        end
      rescue => e
        Rails.logger.error "Payment reconciliation failed for #{billing_payment.id}: #{e.message}"
      end
    end
  end
end

# 3. Monitor payment conversion rates
def payment_health_metrics
  total_payments = BillingPayment.where('created_at > ?', 1.day.ago).count
  successful_payments = BillingPayment.joins(:basic_billing_payment_status)
                                     .where(basic_billing_payment_statuses: { default_name: 'paid' })
                                     .where('billing_payments.created_at > ?', 1.day.ago).count
  
  success_rate = total_payments > 0 ? (successful_payments.to_f / total_payments * 100).round(2) : 0
  
  {
    total_payments: total_payments,
    successful_payments: successful_payments,
    success_rate: "#{success_rate}%",
    failed_payments: total_payments - successful_payments
  }
end

# Test S3 connectivity
begin
  s3_client = Aws::S3::Client.new(
    region: Rails.application.credentials.aws_region,
    access_key_id: Rails.application.credentials.aws_access_key_id,
    secret_access_key: Rails.application.credentials.aws_secret_access_key
  )
  
  # Test bucket access
  response = s3_client.head_bucket(bucket: 'kulcare-uploads')
  puts "S3 bucket accessible: #{response.successful?}"
rescue => e
  puts "S3 error: #{e.message}"
end

# Check recent upload failures
recent_uploads = Document.where('created_at > ?', 1.hour.ago)
failed_uploads = recent_uploads.where(file_url: nil)
puts "Failed uploads: #{failed_uploads.count} / #{recent_uploads.count}"

# 1. Verify AWS credentials and permissions
aws_creds = Rails.application.credentials
puts "AWS Access Key ID: #{aws_creds.aws_access_key_id.present?}"
puts "AWS Secret Key: #{aws_creds.aws_secret_access_key.present?}"
puts "AWS Region: #{aws_creds.aws_region}"

# 2. Implement retry logic for uploads
class S3UploadJob < ApplicationJob
  retry_on Aws::S3::Errors::ServiceError, wait: :exponentially_longer, attempts: 3
  
  def perform(file_path, bucket, key)
    s3_client = Aws::S3::Client.new
    
    s3_client.put_object(
      bucket: bucket,
      key: key,
      body: File.read(file_path),
      content_type: MIME::Types.type_for(file_path).first.content_type
    )
  end
end

# 3. Add file upload monitoring
def s3_health_check
  begin
    s3 = Aws::S3::Client.new
    s3.head_bucket(bucket: Rails.application.credentials.aws_s3_bucket)
    { s3_status: 'healthy', last_check: Time.current }
  rescue => e
    { s3_status: 'error', error: e.message, last_check: Time.current }
  end
end

# Check SNS service status
begin
  sns = Aws::SNS::Client.new
  topics = sns.list_topics
  puts "SNS accessible, topics: #{topics.topics.count}"
rescue => e
  puts "SNS error: #{e.message}"
end

# Check device endpoint status
user = User.find(123)
user.devices.each do |device|
  puts "Device: #{device.device_name}"
  puts "Platform: #{device.device_type}"
  puts "Endpoint ARN: #{device.endpoint_arn.present?}"
  
  if device.endpoint_arn.present?
    begin
      endpoint = sns.get_endpoint_attributes(endpoint_arn: device.endpoint_arn)
      puts "Endpoint enabled: #{endpoint.attributes['Enabled']}"
    rescue => e
      puts "Endpoint error: #{e.message}"
    end
  end
end

# Check recent push notification failures  
failed_notifications = PushNotificationLog.where(status: 'failed')
                                          .where('created_at > ?', 1.hour.ago)
puts "Failed notifications: #{failed_notifications.count}"

# 1. Implement automatic endpoint cleanup
class CleanupSnsEndpoints < ApplicationJob
  def perform
    Device.where.not(endpoint_arn: nil).find_each do |device|
      begin
        sns = Aws::SNS::Client.new
        attrs = sns.get_endpoint_attributes(endpoint_arn: device.endpoint_arn)
        
        # Remove disabled endpoints
        unless attrs.attributes['Enabled'] == 'true'
          device.update!(endpoint_arn: nil)
          Rails.logger.info "Removed disabled endpoint for device #{device.id}"
        end
      rescue Aws::SNS::Errors::NotFound
        # Endpoint doesn't exist, clean up local reference
        device.update!(endpoint_arn: nil)
      rescue => e
        Rails.logger.error "Endpoint cleanup error: #{e.message}"
      end
    end
  end
end

# 2. Add push notification retry logic
class DeferPushNotification < ApplicationJob
  retry_on Aws::SNS::Errors::ServiceError, wait: :exponentially_longer, attempts: 3
  
  def perform(*args)
    result = PushNotifications.send(*args)
    
    # Create endpoint if missing and retry
    if result.nil? && args[0].present?  # user_id present
      user = User.find(args[0])
      user.devices.where(endpoint_arn: nil).each do |device|
        SnsCreatePlatformEndpoint.perform_now(device.id)
      end
      
      # Retry with new endpoint
      PushNotifications.send(*args)
    end
  end
end

# 3. Monitor push notification health
def push_notification_health
  {
    active_devices: Device.where.not(endpoint_arn: nil).count,
    notifications_sent_today: count_notifications_today,
    notification_success_rate: calculate_success_rate,
    disabled_endpoints: count_disabled_endpoints
  }
end

# Check GMB account tokens
gmb_account = GmbAccount.find(123)
puts "Access token present: #{gmb_account.access_token.present?}"
puts "Token expires at: #{Time.at(gmb_account.expires_at)}" if gmb_account.expires_at
puts "Token expired: #{gmb_account.expires_at < Time.current.to_i}"

# Test GMB API connectivity
begin
  headers = {
    'Authorization' => "Bearer #{gmb_account.access_token}",
    'Content-Type' => 'application/json'
  }
  
  response = HTTParty.get(
    "https://mybusinessbusinessinformation.googleapis.com/v1/accounts",
    headers: headers
  )
  
  puts "GMB API Status: #{response.code}"
  puts "Response: #{response.body}" if response.code != 200
rescue => e
  puts "GMB API Error: #{e.message}"
end

# 1. Implement token refresh mechanism
class RefreshGmbTokens < ApplicationJob
  def perform
    GmbAccount.where('expires_at < ?', 1.hour.from_now.to_i).find_each do |account|
      begin
        # Refresh token using Google OAuth
        new_token = refresh_google_token(account.refresh_token)
        
        account.update!(
          access_token: new_token['access_token'],
          expires_at: Time.current.to_i + new_token['expires_in']
        )
        
        Rails.logger.info "Refreshed GMB token for account #{account.id}"
      rescue => e
        Rails.logger.error "Token refresh failed for GMB account #{account.id}: #{e.message}"
      end
    end
  end
  
  private
  
  def refresh_google_token(refresh_token)
    response = HTTParty.post('https://oauth2.googleapis.com/token', {
      body: {
        client_id: Rails.application.credentials.google_client_id,
        client_secret: Rails.application.credentials.google_client_secret,
        refresh_token: refresh_token,
        grant_type: 'refresh_token'
      }
    })
    
    JSON.parse(response.body)
  end
end

# 2. Handle API rate limits
class GoogleApiService
  def self.with_rate_limiting(&block)
    retry_count = 0
    begin
      yield
    rescue => e
      if e.message.include?('quota') && retry_count < 3
        retry_count += 1
        sleep(2 ** retry_count)  # Exponential backoff
        retry
      else
        raise
      end
    end
  end
end

# 3. Monitor GMB integration health
def gmb_health_check
  {
    active_accounts: GmbAccount.where('expires_at > ?', Time.current.to_i).count,
    expired_tokens: GmbAccount.where('expires_at < ?', Time.current.to_i).count,
    locations_synced_today: gmb_sync_count_today,
    api_errors_today: gmb_error_count_today
  }
end

# Comprehensive integration health check
class IntegrationHealthCheck
  def self.perform
    {
      sms_services: check_sms_services,
      whatsapp_service: check_whatsapp_service,
      payment_gateway: check_payment_gateway,
      aws_services: check_aws_services,
      google_services: check_google_services,
      overall_status: calculate_overall_status
    }
  end
  
  private
  
  def self.check_sms_services
    {
      msg91: test_msg91_connectivity,
      twilio: test_twilio_connectivity,
      delivery_rate_24h: calculate_sms_delivery_rate
    }
  end
  
  def self.check_whatsapp_service
    {
      wati_api: test_wati_connectivity,
      webhook_match_rate: calculate_webhook_match_rate,
      delivery_rate_24h: calculate_whatsapp_delivery_rate
    }
  end
  
  # Additional service checks...
end

# Schedule health checks
# In config/schedule.rb (whenever gem)
every 15.minutes do
  runner "IntegrationHealthCheck.perform"
end

# Integration alert system
class IntegrationAlerts
  ALERT_THRESHOLDS = {
    sms_delivery_rate: 85,
    whatsapp_delivery_rate: 80,
    payment_success_rate: 95,
    api_response_time: 5000  # milliseconds
  }.freeze
  
  def self.check_and_alert
    health_status = IntegrationHealthCheck.perform
    
    ALERT_THRESHOLDS.each do |metric, threshold|
      current_value = extract_metric_value(health_status, metric)
      
      if current_value < threshold
        send_alert(
          "#{metric} below threshold: #{current_value}% (threshold: #{threshold}%)"
        )
      end
    end
  end
  
  private
  
  def self.send_alert(message)
    SlackAlerts.perform_later(message)
    # Also send to monitoring service (PagerDuty, etc.)
  end
end

# Implement circuit breaker for external services
class CircuitBreaker
  def initialize(service_name, failure_threshold: 5, recovery_timeout: 60)
    @service_name = service_name
    @failure_threshold = failure_threshold
    @recovery_timeout = recovery_timeout
    @failure_count = 0
    @last_failure_time = nil
    @state = :closed
  end
  
  def call(&block)
    case @state
    when :open
      if time_to_retry?
        @state = :half_open
      else
        raise CircuitBreakerOpenError, "Circuit breaker is open for #{@service_name}"
      end
    end
    
    begin
      result = yield
      reset_failures if @state == :half_open
      result
    rescue => e
      record_failure
      raise
    end
  end
  
  private
  
  def record_failure
    @failure_count += 1
    @last_failure_time = Time.current
    
    if @failure_count >= @failure_threshold
      @state = :open
    end
  end
  
  def reset_failures
    @failure_count = 0
    @state = :closed
  end
  
  def time_to_retry?
    Time.current - @last_failure_time > @recovery_timeout
  end
end

# Implement fallback strategies
class CommunicationService
  def send_notification(phone_number, message)
    # Try WhatsApp first
    begin
      WhatsApp::Base.perform_now(phone_number, template, params)
    rescue => whatsapp_error
      Rails.logger.warn "WhatsApp failed, falling back to SMS: #{whatsapp_error.message}"
      
      # Fallback to SMS
      begin
        Sms::Base.perform_now(phone_number, message, dlt_template_id)
      rescue => sms_error
        Rails.logger.error "Both WhatsApp and SMS failed: #{sms_error.message}"
        
        # Final fallback - queue for later retry
        RetryNotificationJob.set(wait: 10.minutes).perform_later(
          phone_number, message
        )
      end
    end
  end
end

# Restart integration-related services
sudo systemctl restart sidekiq  # Background jobs
sudo systemctl restart puma     # Web server
sudo systemctl restart redis    # Cache/sessions

# Clear stuck jobs
bundle exec rails c
> Sidekiq::Queue.new('sms_messages').clear
> Sidekiq::Queue.new('whatsapp_messages').clear
> Sidekiq::RetrySet.new.clear

# Clear failed integration logs
MsgLog.where(msg_status: 'FAILED').where('created_at < ?', 1.week.ago).destroy_all
WatiWebhook.where('created_at < ?', 1.month.ago).destroy_all
RazorpayWebhookError.where('created_at < ?', 1.month.ago).destroy_all

# Reset stuck payment statuses
BillingPayment.where(razorpay_payment_id: nil)
              .where('created_at < ?', 1.hour.ago)
              .update_all(status: 'failed')

# Reset integration configurations
# In Rails console
Rails.application.credentials.config  # View all credentials

# Test credential access
puts Rails.application.credentials.msg91_auth_key.present?
puts Rails.application.credentials.razorpay_key_id.present?
puts Rails.application.credentials.wati.api_token.present?