Monitoring and Logging Setup

Rails.application.configure do
  # Logging configuration
  config.log_level = ENV.fetch('LOG_LEVEL', 'info').to_sym
  config.log_tags = [:request_id, :remote_ip, :user_id]
  
  # Use JSON formatter for structured logging
  if ENV['STRUCTURED_LOGGING'] == 'true'
    config.log_formatter = JsonLogFormatter.new
  end
  
  # Log to stdout in production (for container/systemd logging)
  config.logger = ActiveSupport::Logger.new(STDOUT)
  
  # Disable Rails asset logging in production
  config.assets.quiet = true
  
  # Filter sensitive parameters from logs
  config.filter_parameters += [
    :password, :password_confirmation, :token, :api_key,
    :authorization, :authentication, :secret, :otp_code,
    :phone_number, :email, :aadhaar_number, :medical_data
  ]
end

class JsonLogFormatter < ActiveSupport::Logger::SimpleFormatter
  def call(severity, timestamp, progname, msg)
    log_entry = {
      timestamp: timestamp.iso8601,
      level: severity,
      message: extract_message(msg),
      service: 'kulcare-api-backend',
      environment: Rails.env
    }
    
    # Add request context if available
    if Current.request_id
      log_entry[:request_id] = Current.request_id
      log_entry[:user_id] = Current.user&.id
      log_entry[:clinic_id] = Current.clinic&.id
    end
    
    "#{log_entry.to_json}\n"
  end
  
  private
  
  def extract_message(msg)
    case msg
    when String
      msg
    when Hash
      msg
    else
      msg.to_s
    end
  end
end

class RequestLoggingMiddleware
  def initialize(app)
    @app = app
  end

  def call(env)
    request = ActionDispatch::Request.new(env)
    request_id = request.headers['X-Request-ID'] || SecureRandom.uuid
    
    # Set request context
    Current.request_id = request_id
    env['action_dispatch.request_id'] = request_id
    
    # Log request start
    start_time = Time.current
    log_request_start(request, request_id)
    
    # Process request
    status, headers, response = @app.call(env)
    
    # Log request completion
    duration = (Time.current - start_time) * 1000
    log_request_complete(request, status, duration, request_id)
    
    [status, headers, response]
  rescue => e
    # Log request error
    log_request_error(request, e, request_id)
    raise
  ensure
    # Clear request context
    Current.request_id = nil
  end

  private

  def log_request_start(request, request_id)
    Rails.logger.info({
      event: 'request_started',
      method: request.method,
      path: request.path,
      params: filtered_params(request),
      user_agent: request.headers['User-Agent'],
      remote_ip: request.remote_ip,
      request_id: request_id
    })
  end

  def log_request_complete(request, status, duration, request_id)
    Rails.logger.info({
      event: 'request_completed',
      method: request.method,
      path: request.path,
      status: status,
      duration_ms: duration.round(2),
      request_id: request_id
    })
  end

  def log_request_error(request, error, request_id)
    Rails.logger.error({
      event: 'request_error',
      method: request.method,
      path: request.path,
      error_class: error.class.name,
      error_message: error.message,
      backtrace: error.backtrace&.first(10),
      request_id: request_id
    })
  end

  def filtered_params(request)
    params = request.params.except('controller', 'action')
    Rails.application.config.filter_parameters.each do |filter|
      filter_param(params, filter)
    end
    params
  end

  def filter_param(params, filter)
    params.each do |key, value|
      if key.to_s.match?(filter)
        params[key] = '[FILTERED]'
      elsif value.is_a?(Hash)
        filter_param(value, filter)
      end
    end
  end
end

config.middleware.use RequestLoggingMiddleware

if Rails.env.production? && ENV['LOG_QUERIES'] == 'true'
  ActiveSupport::Notifications.subscribe 'sql.active_record' do |name, start, finish, id, payload|
    duration = (finish - start) * 1000
    
    # Log slow queries
    if duration > 100 # Log queries slower than 100ms
      Rails.logger.warn({
        event: 'slow_query',
        sql: payload[:sql],
        duration_ms: duration.round(2),
        name: payload[:name],
        connection_id: payload[:connection_id]
      })
    end
  end
end

# Gemfile
gem 'sentry-ruby'
gem 'sentry-rails'

Sentry.init do |config|
  config.dsn = ENV['SENTRY_DSN']
  config.breadcrumbs_logger = [:active_support_logger, :http_logger]
  
  # Set environment
  config.environment = Rails.env
  config.release = ENV['APP_VERSION'] || 'unknown'
  
  # Performance monitoring
  config.traces_sample_rate = Rails.env.production? ? 0.1 : 1.0
  
  # Filter sensitive data
  config.before_send = lambda do |event, hint|
    # Filter sensitive parameters
    if event.request && event.request.data
      event.request.data = filter_sensitive_data(event.request.data)
    end
    
    # Add user context
    if Current.user
      event.user = {
        id: Current.user.id,
        email: Current.user.email,
        clinic_id: Current.clinic&.id
      }
    end
    
    event
  end
end

def filter_sensitive_data(data)
  sensitive_keys = %w[password token api_key phone_number email aadhaar_number]
  
  data.each do |key, value|
    if sensitive_keys.any? { |sensitive| key.to_s.include?(sensitive) }
      data[key] = '[FILTERED]'
    elsif value.is_a?(Hash)
      data[key] = filter_sensitive_data(value)
    end
  end
  
  data
end

class HealthController < ApplicationController
  skip_before_action :authenticate_user!
  
  def index
    render json: {
      status: 'healthy',
      timestamp: Time.current.iso8601,
      version: ENV['APP_VERSION'] || 'unknown'
    }
  end
  
  def detailed
    checks = {
      database: database_check,
      redis: redis_check,
      elasticsearch: elasticsearch_check,
      sidekiq: sidekiq_check,
      external_apis: external_apis_check
    }
    
    overall_status = checks.values.all? { |check| check[:status] == 'healthy' }
    
    render json: {
      status: overall_status ? 'healthy' : 'unhealthy',
      checks: checks,
      timestamp: Time.current.iso8601
    }, status: overall_status ? :ok : :service_unavailable
  end
  
  private
  
  def database_check
    ActiveRecord::Base.connection.execute('SELECT 1')
    { status: 'healthy', response_time: measure_time { ActiveRecord::Base.connection.execute('SELECT 1') } }
  rescue => e
    { status: 'unhealthy', error: e.message }
  end
  
  def redis_check
    Redis.new(url: ENV['REDIS_URL']).ping
    { status: 'healthy', response_time: measure_time { Redis.new(url: ENV['REDIS_URL']).ping } }
  rescue => e
    { status: 'unhealthy', error: e.message }
  end
  
  def elasticsearch_check
    client = Elasticsearch::Client.new(url: ENV['ELASTICSEARCH_URL'])
    client.cluster.health
    { status: 'healthy', response_time: measure_time { client.cluster.health } }
  rescue => e
    { status: 'unhealthy', error: e.message }
  end
  
  def sidekiq_check
    stats = Sidekiq::Stats.new
    healthy = stats.processes_size > 0
    
    {
      status: healthy ? 'healthy' : 'unhealthy',
      processes: stats.processes_size,
      enqueued: stats.enqueued,
      failed: stats.failed
    }
  rescue => e
    { status: 'unhealthy', error: e.message }
  end
  
  def external_apis_check
    checks = {
      twilio: check_twilio,
      razorpay: check_razorpay,
      aws_s3: check_s3
    }
    
    overall_healthy = checks.values.all? { |check| check[:status] == 'healthy' }
    
    {
      status: overall_healthy ? 'healthy' : 'partial',
      services: checks
    }
  end
  
  def check_twilio
    # Basic Twilio API check
    account_sid = ENV['TWILIO_ACCOUNT_SID']
    return { status: 'disabled' } unless account_sid
    
    response_time = measure_time do
      HTTParty.get("https://api.twilio.com/2010-04-01/Accounts/#{account_sid}.json",
                   basic_auth: { username: account_sid, password: ENV['TWILIO_AUTH_TOKEN'] })
    end
    
    { status: 'healthy', response_time: response_time }
  rescue => e
    { status: 'unhealthy', error: e.message }
  end
  
  def check_razorpay
    # Razorpay API check
    return { status: 'disabled' } unless ENV['RAZORPAY_KEY_ID']
    
    response_time = measure_time do
      HTTParty.get('https://api.razorpay.com/v1/payments',
                   basic_auth: { username: ENV['RAZORPAY_KEY_ID'], password: ENV['RAZORPAY_KEY_SECRET'] })
    end
    
    { status: 'healthy', response_time: response_time }
  rescue => e
    { status: 'unhealthy', error: e.message }
  end
  
  def check_s3
    return { status: 'disabled' } unless ENV['AWS_S3_BUCKET']
    
    response_time = measure_time do
      s3 = Aws::S3::Client.new
      s3.head_bucket(bucket: ENV['AWS_S3_BUCKET'])
    end
    
    { status: 'healthy', response_time: response_time }
  rescue => e
    { status: 'unhealthy', error: e.message }
  end
  
  def measure_time(&block)
    start_time = Time.current
    block.call
    ((Time.current - start_time) * 1000).round(2)
  end
end

get '/health', to: 'health#index'
get '/health/detailed', to: 'health#detailed'

/var/www/kulcare-api-backend/log/*.log {
    daily
    missingok
    rotate 30
    compress
    delaycompress
    notifempty
    create 644 kulcare kulcare
    sharedscripts
    postrotate
        /bin/kill -USR1 $(cat /var/www/kulcare-api-backend/tmp/pids/puma.pid) > /dev/null 2>&1 || true
    endscript
}

/var/log/nginx/*.log {
    daily
    missingok
    rotate 30
    compress
    delaycompress
    notifempty
    create 644 www-data www-data
    sharedscripts
    postrotate
        /bin/kill -USR1 $(cat /var/run/nginx.pid) > /dev/null 2>&1 || true
    endscript
}

#!/bin/bash

# System monitoring script
LOG_FILE="/var/log/kulcare/system_monitor.log"
ALERT_EMAIL="admin@kulcare.com"
HOSTNAME=$(hostname)

# Ensure log directory exists
mkdir -p /var/log/kulcare

# Function to log with timestamp
log_with_timestamp() {
    echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> $LOG_FILE
}

# Function to send alert
send_alert() {
    local subject="$1"
    local message="$2"
    
    # Log the alert
    log_with_timestamp "ALERT: $subject - $message"
    
    # Send email (if mail is configured)
    if command -v mail >/dev/null 2>&1; then
        echo "$message" | mail -s "[$HOSTNAME] $subject" $ALERT_EMAIL
    fi
}

# Check disk space
check_disk_space() {
    local usage=$(df / | awk 'NR==2 {print $5}' | sed 's/%//')
    
    if [ $usage -gt 90 ]; then
        send_alert "High Disk Usage" "Disk usage is at ${usage}%"
    elif [ $usage -gt 80 ]; then
        log_with_timestamp "WARNING: Disk usage is at ${usage}%"
    fi
}

# Check memory usage
check_memory_usage() {
    local mem_usage=$(free | awk 'NR==2{printf "%.1f", $3*100/$2}')
    local mem_usage_int=${mem_usage%.*}
    
    if [ $mem_usage_int -gt 90 ]; then
        send_alert "High Memory Usage" "Memory usage is at ${mem_usage}%"
    elif [ $mem_usage_int -gt 80 ]; then
        log_with_timestamp "WARNING: Memory usage is at ${mem_usage}%"
    fi
}

# Check CPU load
check_cpu_load() {
    local load_avg=$(uptime | awk -F'load average:' '{print $2}' | awk '{print $1}' | sed 's/,//')
    local load_int=${load_avg%.*}
    local cpu_cores=$(nproc)
    
    if (( $(echo "$load_avg > $cpu_cores * 2" | bc -l) )); then
        send_alert "High CPU Load" "Load average is $load_avg (${cpu_cores} cores available)"
    fi
}

# Check critical processes
check_processes() {
    local processes=("puma" "sidekiq" "nginx" "postgresql" "redis-server")
    
    for process in "${processes[@]}"; do
        if ! pgrep -f $process > /dev/null; then
            send_alert "Process Down" "$process is not running"
        fi
    done
}

# Check log file sizes
check_log_sizes() {
    local max_size_mb=500
    local log_dirs=("/var/www/kulcare-api-backend/log" "/var/log/nginx" "/var/log/postgresql")
    
    for log_dir in "${log_dirs[@]}"; do
        if [ -d "$log_dir" ]; then
            find "$log_dir" -name "*.log" -size +${max_size_mb}M -exec basename {} \; | while read log_file; do
                send_alert "Large Log File" "$log_file is larger than ${max_size_mb}MB"
            done
        fi
    done
}

# Check SSL certificate expiry
check_ssl_cert() {
    local domain="api.kulcare.com"
    local days_until_expiry=$(echo | openssl s_client -servername $domain -connect $domain:443 2>/dev/null | openssl x509 -noout -dates | grep 'notAfter' | sed 's/notAfter=//' | xargs -I {} date -d {} +%s)
    local current_date=$(date +%s)
    local days_left=$(( (days_until_expiry - current_date) / 86400 ))
    
    if [ $days_left -lt 30 ]; then
        send_alert "SSL Certificate Expiring" "SSL certificate for $domain expires in $days_left days"
    fi
}

# Run all checks
log_with_timestamp "Starting system monitoring checks"
check_disk_space
check_memory_usage
check_cpu_load
check_processes
check_log_sizes
check_ssl_cert
log_with_timestamp "System monitoring checks completed"

chmod +x /home/kulcare/monitor_system.sh

# Add to crontab (run every 5 minutes)
crontab -e
# Add: */5 * * * * /home/kulcare/monitor_system.sh

#!/bin/bash

LOG_FILE="/var/log/kulcare/db_monitor.log"
DB_NAME="kulcare_production"
DB_USER="kulcare"

log_with_timestamp() {
    echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> $LOG_FILE
}

# Check database connection
check_db_connection() {
    if psql -U $DB_USER -d $DB_NAME -c "SELECT 1;" > /dev/null 2>&1; then
        log_with_timestamp "Database connection: OK"
    else
        log_with_timestamp "ERROR: Database connection failed"
        send_alert "Database Connection Failed" "Cannot connect to $DB_NAME"
    fi
}

# Check database size
check_db_size() {
    local db_size=$(psql -U $DB_USER -d $DB_NAME -t -c "SELECT pg_size_pretty(pg_database_size('$DB_NAME'));")
    log_with_timestamp "Database size: $db_size"
}

# Check slow queries
check_slow_queries() {
    local slow_queries=$(psql -U $DB_USER -d $DB_NAME -t -c "
        SELECT count(*) FROM pg_stat_activity 
        WHERE state = 'active' AND query_start < now() - interval '5 minutes';
    ")
    
    if [ $slow_queries -gt 5 ]; then
        log_with_timestamp "WARNING: $slow_queries slow queries detected"
        send_alert "Slow Queries Detected" "$slow_queries queries running for more than 5 minutes"
    fi
}

# Check database locks
check_locks() {
    local locks=$(psql -U $DB_USER -d $DB_NAME -t -c "
        SELECT count(*) FROM pg_locks WHERE NOT granted;
    ")
    
    if [ $locks -gt 0 ]; then
        log_with_timestamp "WARNING: $locks database locks detected"
    fi
}

# Run database checks
log_with_timestamp "Starting database monitoring checks"
check_db_connection
check_db_size
check_slow_queries
check_locks
log_with_timestamp "Database monitoring checks completed"

# Gemfile
gem 'newrelic_rpm' # or 'scout_apm' or 'skylight'

common: &default_settings
  license_key: '<%= ENV["NEW_RELIC_LICENSE_KEY"] %>'
  app_name: 'Kulcare API Backend'
  
  # Transaction tracing
  transaction_tracer:
    enabled: true
    transaction_threshold: apdex_f
    record_sql: obfuscated
    
  # Error collection
  error_collector:
    enabled: true
    capture_source: true
    ignore_errors: "ActionController::RoutingError,ActionController::InvalidAuthenticityToken"

production:
  <<: *default_settings
  app_name: 'Kulcare API Backend (Production)'
  monitor_mode: true

staging:
  <<: *default_settings
  app_name: 'Kulcare API Backend (Staging)'
  monitor_mode: true

development:
  <<: *default_settings
  monitor_mode: false

test:
  <<: *default_settings
  monitor_mode: false

class MetricsService
  def self.increment(metric_name, tags = {})
    Rails.logger.info({
      event: 'metric',
      type: 'counter',
      name: metric_name,
      value: 1,
      tags: tags
    })
    
    # Send to APM service
    if defined?(NewRelic)
      NewRelic::Agent.increment_metric(metric_name, 1)
    end
  end
  
  def self.timing(metric_name, duration_ms, tags = {})
    Rails.logger.info({
      event: 'metric',
      type: 'timing',
      name: metric_name,
      value: duration_ms,
      tags: tags
    })
    
    # Send to APM service
    if defined?(NewRelic)
      NewRelic::Agent.record_metric(metric_name, duration_ms)
    end
  end
  
  def self.gauge(metric_name, value, tags = {})
    Rails.logger.info({
      event: 'metric',
      type: 'gauge',
      name: metric_name,
      value: value,
      tags: tags
    })
  end
end

class AppointmentsController < ApplicationController
  def create
    start_time = Time.current
    
    result = CreateAppointment.call(appointment_params)
    
    # Record metrics
    MetricsService.timing(
      'appointment.create.duration',
      (Time.current - start_time) * 1000,
      { clinic_id: current_clinic.id, success: result.success? }
    )
    
    if result.success?
      MetricsService.increment('appointment.created', { clinic_id: current_clinic.id })
    else
      MetricsService.increment('appointment.create_failed', { clinic_id: current_clinic.id })
    end
    
    # ... rest of controller logic
  end
end

class AlertService
  ALERT_LEVELS = %w[info warning error critical].freeze
  
  def self.send_alert(level, title, message, metadata = {})
    return unless ALERT_LEVELS.include?(level.to_s)
    
    alert_data = {
      level: level,
      title: title,
      message: message,
      metadata: metadata,
      timestamp: Time.current.iso8601,
      hostname: ENV['HOSTNAME'] || `hostname`.strip,
      service: 'kulcare-api-backend'
    }
    
    # Log the alert
    Rails.logger.send(level == 'critical' ? :error : level, {
      event: 'alert',
      **alert_data
    })
    
    # Send to external services
    send_to_slack(alert_data) if ENV['SLACK_WEBHOOK_URL'] && %w[error critical].include?(level)
    send_email_alert(alert_data) if ENV['ALERT_EMAIL'] && level == 'critical'
    send_to_sentry(alert_data) if defined?(Sentry) && %w[error critical].include?(level)
  end
  
  private
  
  def self.send_to_slack(alert_data)
    webhook_url = ENV['SLACK_WEBHOOK_URL']
    
    payload = {
      text: "#{alert_data[:level].upcase}: #{alert_data[:title]}",
      attachments: [
        {
          color: alert_data[:level] == 'critical' ? 'danger' : 'warning',
          fields: [
            { title: 'Service', value: alert_data[:service], short: true },
            { title: 'Host', value: alert_data[:hostname], short: true },
            { title: 'Message', value: alert_data[:message], short: false }
          ],
          ts: Time.current.to_i
        }
      ]
    }
    
    HTTParty.post(webhook_url, {
      body: payload.to_json,
      headers: { 'Content-Type' => 'application/json' }
    })
  rescue => e
    Rails.logger.error "Failed to send Slack alert: #{e.message}"
  end
  
  def self.send_email_alert(alert_data)
    AlertMailer.critical_alert(alert_data).deliver_later
  rescue => e
    Rails.logger.error "Failed to send email alert: #{e.message}"
  end
  
  def self.send_to_sentry(alert_data)
    Sentry.capture_message(
      "#{alert_data[:title]}: #{alert_data[:message]}",
      level: alert_data[:level] == 'critical' ? :error : :warning,
      extra: alert_data[:metadata]
    )
  end
end

# app/jobs/monitoring_jobs/queue_monitor_job.rb
class QueueMonitorJob < ApplicationJob
  queue_as :monitoring
  
  def perform
    stats = Sidekiq::Stats.new
    
    # Alert on high queue size
    if stats.enqueued > 10000
      AlertService.send_alert(:critical, 
        'High Queue Size', 
        "Sidekiq queue has #{stats.enqueued} jobs enqueued",
        { queue_size: stats.enqueued }
      )
    end
    
    # Alert on high retry count
    if stats.retry_size > 1000
      AlertService.send_alert(:warning,
        'High Retry Queue Size',
        "#{stats.retry_size} jobs in retry queue",
        { retry_size: stats.retry_size }
      )
    end
    
    # Check individual queue latencies
    Sidekiq::Queue.all.each do |queue|
      if queue.latency > 300 # 5 minutes
        AlertService.send_alert(:warning,
          'High Queue Latency',
          "#{queue.name} queue latency is #{queue.latency.round(2)} seconds",
          { queue: queue.name, latency: queue.latency }
        )
      end
    end
  end
end

# config/schedule.yml
queue_monitor:
  cron: "*/5 * * * *"  # Every 5 minutes
  class: "QueueMonitorJob"
  description: "Monitor Sidekiq queue health"

error_rate_monitor:
  cron: "*/10 * * * *"  # Every 10 minutes  
  class: "ErrorRateMonitorJob"
  description: "Monitor application error rates"

filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/www/kulcare-api-backend/log/*.log
  fields:
    service: kulcare-api-backend
    environment: production
  fields_under_root: true
  multiline.pattern: '^\d{4}-\d{2}-\d{2}'
  multiline.negate: true
  multiline.match: after

- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log
    - /var/log/nginx/error.log
  fields:
    service: nginx
    environment: production
  fields_under_root: true

output.logstash:
  hosts: ["logstash-server:5044"]

processors:
  - add_host_metadata:
      when.not.contains.tags: forwarded

#!/bin/bash
# /home/kulcare/cleanup_logs.sh

# Clean up application logs older than 30 days
find /var/www/kulcare-api-backend/log -name "*.log" -mtime +30 -delete
find /var/www/kulcare-api-backend/log -name "*.log.gz" -mtime +90 -delete

# Clean up nginx logs older than 60 days
find /var/log/nginx -name "*.log" -mtime +60 -delete
find /var/log/nginx -name "*.gz" -mtime +90 -delete

# Clean up system logs
journalctl --vacuum-time=30d

echo "Log cleanup completed at $(date)"

# Daily log cleanup at 3 AM
0 3 * * * /home/kulcare/cleanup_logs.sh >> /var/log/kulcare/cleanup.log 2>&1