Difference between revisions of "Zabbix Cleanup"

From UVOO Tech Wiki
Jump to navigation Jump to search
Line 8: Line 8:
 
# tables=( "history_uint" "history" "history_text" "history_str" "history_log" "trends" "trends_uint" )
 
# tables=( "history_uint" "history" "history_text" "history_str" "history_log" "trends" "trends_uint" )
 
tables=( "history_uint" "history" "history_text" "history_str" "history_log" )
 
tables=( "history_uint" "history" "history_text" "history_str" "history_log" )
 +
  
 
timestamp() {
 
timestamp() {
Line 14: Line 15:
 
   echo $ts
 
   echo $ts
 
}
 
}
 +
 +
 +
delete_table_records(){
 +
  table=$1
 +
  echo I: Starting delete of table $table records.
 +
  psql ${db} -c "DELETE FROM ${table}"
 +
  echo I: Ending delete of table $table records.
 +
}
 +
  
 
delete(){
 
delete(){
Line 19: Line 29:
 
   days_before=$2
 
   days_before=$2
 
   echo I: Starting delete of table $table items before ${days_before} in past.
 
   echo I: Starting delete of table $table items before ${days_before} in past.
 +
  timestamp
 
   echo "DELETE FROM ${table} WHERE clock < extract(epoch from (SELECT NOW() - INTERVAL '${days_before} DAY'))"
 
   echo "DELETE FROM ${table} WHERE clock < extract(epoch from (SELECT NOW() - INTERVAL '${days_before} DAY'))"
 +
  time psql ${db} -c "DELETE FROM ${table} WHERE clock < extract(epoch from (SELECT NOW() - INTERVAL '${days_before} DAY'))"
 +
  timestamp
 
   echo I: Ending delete of table $table before ${days_before} in past.
 
   echo I: Ending delete of table $table before ${days_before} in past.
 
}
 
}
 +
  
 
vacuum(){
 
vacuum(){
 
   table=$1
 
   table=$1
 +
  echo I: Starting vacuum of table $table.
 +
  timestamp
 +
  # psql ${db} -c "VACUUM(ANALYZE, VERBOSE) ${table}"
 +
  # psql ${db} -c "VACUUM(FULL, VERBOSE) ${table}"
 +
  time psql ${db} -c "VACUUM(ANALYZE, VERBOSE) ${table}"
 
   timestamp
 
   timestamp
−
  echo I: Starting vacuum of table $table.
 
−
  psql ${db} -c "VACUUM(ANALYZE, VERBOSE) ${table}"
 
 
   echo I: Ending vacuum of table $table.
 
   echo I: Ending vacuum of table $table.
 +
}
 +
 +
 +
reindex(){
 +
  table=$1
 +
  echo I: Starting reindex concurrently of table $table.
 +
  timestamp
 +
  time psql ${db} -c "REINDEX TABLE CONCURRENTLY ${table}"
 +
  timestamp
 +
  echo I: Ending reindex concurrently of table $table.
 +
}
 +
 +
 +
vaccum_full_reindexmain(){
 +
  time psql ${db} -c "VACUUM FULL VERBOSE"
 +
  time psql ${db} -c "REINDEX VERBOSE DATABASE ${db}"
 +
}
 +
 +
 +
vaccum_reindex_db(){
 +
  echo start
 +
  # timestamp
 +
  # psql ${db} -c "VACUUM VERBOSE"
 +
  # psql ${db} -c "REINDEX CONCURRENTLY VERBOSE DATABASE ${db}"
 
}
 
}
  
  
 
main(){
 
main(){
−
 
+
  echo Start table optimize.
 
   timestamp
 
   timestamp
 
   for table in "${tables[@]}"; do
 
   for table in "${tables[@]}"; do
 
     echo $table
 
     echo $table
−
     delete $table 35
+
     delete $table 5
 
     vacuum $table
 
     vacuum $table
 +
    reindex $table
 +
    # delete_table_records $table
 
   done
 
   done
 
   timestamp
 
   timestamp
 +
  echo End table optimize.
 
}
 
}
 +
  
 
main
 
main

Revision as of 16:16, 16 August 2022

Script 

#!/bin/bash
set -eu

db=zabbix

# tables=( "history_uint" "history" "history_text" "history_str" "history_log" "trends" "trends_uint" )
tables=( "history_uint" "history" "history_text" "history_str" "history_log" )


timestamp() {
  #date +"%T"
  ts=$(date "+%Y%m%d-%H%M%S")
  echo $ts
}


delete_table_records(){
  table=$1
  echo I: Starting delete of table $table records.
  psql ${db} -c "DELETE FROM ${table}"
  echo I: Ending delete of table $table records.
}


delete(){
  table=$1
  days_before=$2
  echo I: Starting delete of table $table items before ${days_before} in past.
  timestamp
  echo "DELETE FROM ${table} WHERE clock < extract(epoch from (SELECT NOW() - INTERVAL '${days_before} DAY'))"
  time psql ${db} -c "DELETE FROM ${table} WHERE clock < extract(epoch from (SELECT NOW() - INTERVAL '${days_before} DAY'))"
  timestamp
  echo I: Ending delete of table $table before ${days_before} in past.
}


vacuum(){
  table=$1
  echo I: Starting vacuum of table $table.
  timestamp
  # psql ${db} -c "VACUUM(ANALYZE, VERBOSE) ${table}"
  # psql ${db} -c "VACUUM(FULL, VERBOSE) ${table}"
  time psql ${db} -c "VACUUM(ANALYZE, VERBOSE) ${table}"
  timestamp
  echo I: Ending vacuum of table $table.
}


reindex(){
  table=$1
  echo I: Starting reindex concurrently of table $table.
  timestamp
  time psql ${db} -c "REINDEX TABLE CONCURRENTLY ${table}"
  timestamp
  echo I: Ending reindex concurrently of table $table.
}


vaccum_full_reindexmain(){
  time psql ${db} -c "VACUUM FULL VERBOSE"
  time psql ${db} -c "REINDEX VERBOSE DATABASE ${db}"
}


vaccum_reindex_db(){
  echo start
  # timestamp
  # psql ${db} -c "VACUUM VERBOSE"
  # psql ${db} -c "REINDEX CONCURRENTLY VERBOSE DATABASE ${db}"
}


main(){
  echo Start table optimize.
  timestamp
  for table in "${tables[@]}"; do
    echo $table
    delete $table 5
    vacuum $table
    reindex $table
    # delete_table_records $table
  done
  timestamp
  echo End table optimize.
}


main

DELETE FROM history_uint WHERE clock < extract(epoch from (SELECT NOW() - INTERVAL '35 DAY'));
DELETE FROM history WHERE clock < extract(epoch from (SELECT NOW() - INTERVAL '35 DAY'));

DELETE FROM history_str WHERE clock < extract(epoch from (SELECT NOW() - INTERVAL '35 DAY'));
DELETE FROM history_text WHERE clock < extract(epoch from (SELECT NOW() - INTERVAL '35 DAY'));
DELETE FROM history_log WHERE clock < extract(epoch from (SELECT NOW() - INTERVAL '35 DAY'));

other options
trends_uint
trends
events
alerts

orphans 

select count(*) from item_tag where itemid not in (select itemid from items);
delete from item_tag where itemid not in (select itemid from items);

https://github.com/azmelanar/zabbix-database-cleanup/blob/master/orphaned/clean.sql

https://github.com/burner1024/zabbix-sql

https://www.zabbix.com/documentation/2.2/manual/api/reference/item/object 

Status of the item.

Possible values:
0 - (default) enabled item;
1 - disabled item.

delete from history where itemid not in (select itemid from items where status='0');
delete from history_uint where itemid not in (select itemid from items where status='0');
delete from history_str where itemid not in (select itemid from items where status='0');
delete from history_text where itemid not in (select itemid from items where status='0');
delete from history_log where itemid not in (select itemid from items where status='0');

delete from trends where itemid not in (select itemid from items where status='0');
delete from trends_uint where itemid not in (select itemid from items where status='0');

DELETE FROM auditlog WHERE NOT userid IN (SELECT userid FROM users);

-- Delete orphaned conditions 
DELETE FROM conditions WHERE NOT actionid IN (SELECT actionid FROM actions);

-- Delete orphaned functions 
DELETE FROM functions WHERE NOT itemid IN (SELECT itemid FROM items);
DELETE FROM functions WHERE NOT triggerid IN (SELECT triggerid FROM triggers);

-- Delete orphaned graph items 
DELETE FROM graphs_items WHERE NOT graphid IN (SELECT graphid FROM graphs);
DELETE FROM graphs_items WHERE NOT itemid IN (SELECT itemid FROM items);

-- Delete orphaed host_profiles 
DELETE FROM hosts_profiles WHERE NOT hostid IN (SELECT hostid FROM hosts);
DELETE FROM hosts_profiles_ext WHERE NOT hostid IN (SELECT hostid FROM hosts);

-- Delete orphaned host macro's 
DELETE FROM hostmacro WHERE NOT hostid IN (SELECT hostid FROM hosts);

-- Delete orphaned item data 
DELETE FROM items WHERE hostid NOT IN (SELECT hostid FROM hosts);
DELETE FROM items_applications WHERE applicationid NOT IN (SELECT applicationid FROM applications);
DELETE FROM items_applications WHERE itemid NOT IN (SELECT itemid FROM items);

-- Delete orphaned HTTP check data 
DELETE FROM httpstep WHERE NOT httptestid IN (SELECT httptestid FROM httptest);
DELETE FROM httpstepitem WHERE NOT httpstepid IN (SELECT httpstepid FROM httpstep);
DELETE FROM httpstepitem WHERE NOT itemid IN (SELECT itemid FROM items);
DELETE FROM httptest WHERE applicationid NOT IN (SELECT applicationid FROM applications);

-- Delete orphaned maintenance data 
DELETE FROM maintenances_groups WHERE maintenanceid NOT IN (SELECT maintenanceid FROM maintenances);
DELETE FROM maintenances_groups WHERE groupid NOT IN (SELECT groupid FROM groups);
DELETE FROM maintenances_hosts WHERE maintenanceid NOT IN (SELECT maintenanceid FROM maintenances);
DELETE FROM maintenances_hosts WHERE hostid NOT IN (SELECT hostid FROM hosts);
DELETE FROM maintenances_windows WHERE maintenanceid NOT IN (SELECT maintenanceid FROM maintenances);
DELETE FROM maintenances_windows WHERE timeperiodid NOT IN (SELECT timeperiodid FROM timeperiods);

-- Delete orphaned mappings 
DELETE FROM mappings WHERE NOT valuemapid IN (SELECT valuemapid FROM valuemaps);

-- Delete orphaned media / user items 
DELETE FROM media WHERE NOT userid IN (SELECT userid FROM users);
DELETE FROM media WHERE NOT mediatypeid IN (SELECT mediatypeid FROM media_type);
DELETE FROM rights WHERE NOT groupid IN (SELECT usrgrpid FROM usrgrp);
DELETE FROM rights WHERE NOT id IN (SELECT groupid FROM groups);
DELETE FROM sessions WHERE NOT userid IN (SELECT userid FROM users);
DELETE FROM user_history WHERE NOT userid IN (SELECT userid FROM users);

-- Screens 
DELETE FROM screens_items WHERE screenid NOT IN (SELECT screenid FROM screens);

-- Events & triggers 
DELETE FROM trigger_depends WHERE triggerid_down NOT IN (SELECT triggerid FROM triggers);
DELETE FROM trigger_depends WHERE triggerid_up NOT IN (SELECT triggerid FROM triggers);

-- Count the amount of records in the history/trends table for items that no longer exist
DELETE FROM history WHERE itemid NOT IN (SELECT itemid FROM items);
DELETE FROM history_uint WHERE itemid NOT IN (SELECT itemid FROM items);
DELETE FROM history_log WHERE itemid NOT IN (SELECT itemid FROM items);
DELETE FROM history_str WHERE itemid NOT IN (SELECT itemid FROM items);
DELETE FROM history_sync WHERE itemid NOT IN (SELECT itemid FROM items);
DELETE FROM history_text WHERE itemid NOT IN (SELECT itemid FROM items);

DELETE FROM trends WHERE itemid NOT IN (SELECT itemid FROM items);
DELETE FROM trends_uint WHERE itemid NOT IN (SELECT itemid FROM items);

MYSQL 

DELETE FROM history_uint WHERE clock < UNIX_TIMESTAMP(DATE_SUB(NOW(), INTERVAL 35 DAY));
DELETE FROM history WHERE clock < UNIX_TIMESTAMP(DATE_SUB(NOW(), INTERVAL 35 DAY));
DELETE FROM trends_uint WHERE clock < UNIX_TIMESTAMP(DATE_SUB(NOW(), INTERVAL 35 DAY));
DELETE FROM trends WHERE clock < UNIX_TIMESTAMP(DATE_SUB(NOW(), INTERVAL 35 DAY));
DELETE FROM events WHERE clock < UNIX_TIMESTAMP(DATE_SUB(NOW(), INTERVAL 35 DAY));
DELETE FROM alerts WHERE clock < UNIX_TIMESTAMP(DATE_SUB(NOW(), INTERVAL 35 DAY));

Option 2

-- Delete orphaned alerts entries
DELETE FROM alerts WHERE NOT EXISTS (SELECT 1 FROM actions WHERE alerts.actionid = actions.actionid);
DELETE FROM alerts WHERE NOT EXISTS (SELECT 1 FROM events WHERE alerts.eventid = events.eventid);
DELETE FROM alerts WHERE NOT userid IN (SELECT userid FROM users);
DELETE FROM alerts WHERE NOT mediatypeid IN (SELECT mediatypeid FROM media_type);

-- Delete orphaned application entries that no longer map back to a host
DELETE FROM applications WHERE NOT hostid IN (SELECT hostid FROM hosts);

-- Delete orphaned auditlog details (such as logins)
DELETE FROM auditlog_details WHERE NOT auditid IN (SELECT auditid FROM auditlog);
DELETE FROM auditlog WHERE NOT userid IN (SELECT userid FROM users);

-- Delete orphaned conditions
DELETE FROM conditions WHERE NOT actionid IN (SELECT actionid FROM actions);

-- Delete orphaned functions
DELETE FROM functions WHERE NOT itemid IN (SELECT itemid FROM items);
DELETE FROM functions WHERE NOT triggerid IN (SELECT triggerid FROM triggers);

-- Delete orphaned graph items
DELETE FROM graphs_items WHERE NOT graphid IN (SELECT graphid FROM graphs);
DELETE FROM graphs_items WHERE NOT itemid IN (SELECT itemid FROM items);

-- Delete orphaned host macro's
DELETE FROM hostmacro WHERE NOT hostid IN (SELECT hostid FROM hosts);

-- Delete orphaned item data
DELETE FROM items WHERE hostid NOT IN (SELECT hostid FROM hosts);
DELETE FROM items_applications WHERE applicationid NOT IN (SELECT applicationid FROM applications);
DELETE FROM items_applications WHERE itemid NOT IN (SELECT itemid FROM items);

-- Delete orphaned HTTP check data
DELETE FROM httpstep WHERE NOT httptestid IN (SELECT httptestid FROM httptest);
DELETE FROM httpstepitem WHERE NOT httpstepid IN (SELECT httpstepid FROM httpstep);
DELETE FROM httpstepitem WHERE NOT itemid IN (SELECT itemid FROM items);
DELETE FROM httptest WHERE applicationid NOT IN (SELECT applicationid FROM applications);

-- Delete orphaned maintenance data
DELETE FROM maintenances_groups WHERE maintenanceid NOT IN (SELECT maintenanceid FROM maintenances);
DELETE FROM maintenances_groups WHERE groupid NOT IN (SELECT groupid FROM groups);
DELETE FROM maintenances_hosts WHERE maintenanceid NOT IN (SELECT maintenanceid FROM maintenances);
DELETE FROM maintenances_hosts WHERE hostid NOT IN (SELECT hostid FROM hosts);
DELETE FROM maintenances_windows WHERE maintenanceid NOT IN (SELECT maintenanceid FROM maintenances);
DELETE FROM maintenances_windows WHERE timeperiodid NOT IN (SELECT timeperiodid FROM timeperiods);

-- Delete orphaned mappings
DELETE FROM mappings WHERE NOT valuemapid IN (SELECT valuemapid FROM valuemaps);

-- Delete orphaned media items
DELETE FROM media WHERE NOT userid IN (SELECT userid FROM users);
DELETE FROM media WHERE NOT mediatypeid IN (SELECT mediatypeid FROM media_type);
DELETE FROM rights WHERE NOT groupid IN (SELECT usrgrpid FROM usrgrp);
DELETE FROM rights WHERE NOT id IN (SELECT groupid FROM groups);
DELETE FROM sessions WHERE NOT userid IN (SELECT userid FROM users);

-- Delete orphaned screens
DELETE FROM screens_items WHERE screenid NOT IN (SELECT screenid FROM screens);

-- Delete orphaned events & triggers
DELETE FROM trigger_depends WHERE triggerid_down NOT IN (SELECT triggerid FROM triggers);
DELETE FROM trigger_depends WHERE triggerid_up NOT IN (SELECT triggerid FROM triggers);

-- Delete records in the history/trends table for items that no longer exist
DELETE FROM history WHERE NOT EXISTS (SELECT 1 FROM items WHERE history.itemid = items.itemid);
DELETE FROM history_uint WHERE NOT EXISTS (SELECT 1 FROM items WHERE history_uint.itemid = items.itemid);
DELETE FROM history_log WHERE NOT EXISTS (SELECT 1 FROM items WHERE history_log.itemid = items.itemid);
DELETE FROM history_str WHERE NOT EXISTS (SELECT 1 FROM items WHERE history_str.itemid = items.itemid);
DELETE FROM history_text WHERE NOT EXISTS (SELECT 1 FROM items WHERE history_text.itemid = items.itemid);

DELETE FROM trends WHERE NOT EXISTS (SELECT 1 FROM items WHERE trends.itemid = items.itemid);
DELETE FROM trends_uint WHERE NOT EXISTS (SELECT 1 FROM items WHERE trends_uint.itemid = items.itemid);

-- Delete records in the events table for triggers/items that no longer exist
DELETE FROM events WHERE source = 0 AND object = 0 AND objectid NOT IN (SELECT triggerid FROM triggers);
DELETE FROM events WHERE source = 3 AND object = 0 AND objectid NOT IN (SELECT triggerid FROM triggers);
DELETE FROM events WHERE source = 3 AND object = 4 AND objectid NOT IN (SELECT itemid FROM items);

-- Delete all orphaned acknowledge entries
DELETE FROM acknowledges WHERE NOT eventid IN (SELECT eventid FROM events);
DELETE FROM acknowledges WHERE NOT userid IN (SELECT userid FROM users);
DELETE FROM acknowledges WHERE eventid IN (SELECT eventid FROM events WHERE (source = 0 OR source=3) AND object = 0 AND objectid NOT IN (SELECT triggerid FROM triggers));
DELETE FROM acknowledges WHERE eventid IN (SELECT eventid FROM events WHERE source=3 AND object = 4 AND objectid NOT IN (SELECT itemid FROM items));

NOTES

https://www.zabbix.com/forum/zabbix-troubleshooting-and-problems/373410-zabbix-housekeeper-processes-more-than-75-busy 

#!/bin/sh
( flock -n 9 || exit 1
psql -U postgres zabbix -c 'DELETE FROM trends_uint t WHERE ctid IN ( SELECT t.ctid FROM trends_uint t LEFT JOIN items i ON i.itemid = t.itemid WHERE to_timestamp(t.clock) < (current_date - ((i.trends)::interval)) LIMIT 10000);'
psql -U postgres zabbix -c 'DELETE FROM history_str h WHERE ctid IN ( SELECT h.ctid FROM history_str h LEFT JOIN items i ON i.itemid = h.itemid WHERE to_timestamp(h.clock) < (current_date - ((i.history)::interval)) LIMIT 10000);'
psql -U postgres zabbix -c 'DELETE FROM history_text h WHERE ctid IN ( SELECT h.ctid FROM history_text h LEFT JOIN items i ON i.itemid = h.itemid WHERE to_timestamp(h.clock) < (current_date - ((i.history)::interval)) LIMIT 100000);'
psql -U postgres zabbix -c 'DELETE FROM history h WHERE ctid IN ( SELECT h.ctid FROM history h LEFT JOIN items i ON i.itemid = h.itemid WHERE to_timestamp(h.clock) < (current_date - ((i.history)::interval)) LIMIT 300000);'
psql -U postgres zabbix -c 'DELETE FROM history_uint h WHERE ctid IN ( SELECT h.ctid FROM history_uint h LEFT JOIN items i ON i.itemid = h.itemid WHERE to_timestamp(h.clock) < (current_date - ((i.history)::interval)) LIMIT 500000);'
) 9>/var/lock/00-zabbix-history-purge.lock
Retrieved from "https://tech.uvoo.io/index.php?title=Zabbix_Cleanup&oldid=3339"