forked from Mapan/odoo17e
264 lines
12 KiB
Python
264 lines
12 KiB
Python
# -*- coding: utf-8 -*-
|
|
# Part of Odoo. See LICENSE file for full copyright and licensing details.
|
|
|
|
import ast
|
|
from dateutil.relativedelta import relativedelta
|
|
from psycopg2 import sql
|
|
|
|
from odoo import models, api, fields, _
|
|
from odoo.tools import split_every
|
|
|
|
# When cleaning_mode = automatic, _clean_records calls action_validate.
|
|
# This is quite slow so requires smaller batch size.
|
|
DR_CREATE_STEP_AUTO = 5000
|
|
DR_CREATE_STEP_MANUAL = 50000
|
|
|
|
|
|
class DataCleaningModel(models.Model):
|
|
_name = 'data_cleaning.model'
|
|
_description = 'Cleaning Model'
|
|
_order = 'name'
|
|
|
|
active = fields.Boolean(default=True)
|
|
name = fields.Char(
|
|
compute='_compute_name', string='Name', readonly=False, store=True, required=True, copy=True)
|
|
|
|
res_model_id = fields.Many2one('ir.model', string='Model', required=True, ondelete='cascade')
|
|
res_model_name = fields.Char(
|
|
related='res_model_id.model', string='Model Name', readonly=True, store=True)
|
|
|
|
cleaning_mode = fields.Selection([
|
|
('manual', 'Manual'),
|
|
('automatic', 'Automatic'),
|
|
], string='Cleaning Mode', default='manual', required=True)
|
|
|
|
rule_ids = fields.One2many('data_cleaning.rule', 'cleaning_model_id', string='Rules')
|
|
records_to_clean_count = fields.Integer('Records To Clean', compute='_compute_records_to_clean')
|
|
|
|
# User Notifications for Manual clean
|
|
notify_user_ids = fields.Many2many(
|
|
'res.users', string='Notify Users',
|
|
domain=lambda self: [('groups_id', 'in', self.env.ref('base.group_system').id)],
|
|
default=lambda self: self.env.user,
|
|
help='List of users to notify when there are new records to clean')
|
|
notify_frequency = fields.Integer(string='Notify', default=1)
|
|
notify_frequency_period = fields.Selection([
|
|
('days', 'Days'),
|
|
('weeks', 'Weeks'),
|
|
('months', 'Months')], string='Notify Frequency Period', default='weeks')
|
|
last_notification = fields.Datetime(readonly=True)
|
|
|
|
_sql_constraints = [
|
|
('check_notif_freq', 'CHECK(notify_frequency > 0)', 'The notification frequency should be greater than 0'),
|
|
]
|
|
|
|
@api.onchange('res_model_id')
|
|
def _compute_name(self):
|
|
for cm_model in self:
|
|
if not cm_model.name:
|
|
cm_model.name = cm_model.res_model_id.name if cm_model.res_model_id else ''
|
|
|
|
@api.onchange('res_model_id')
|
|
def _onchange_res_model_id(self):
|
|
self.ensure_one()
|
|
if any(rule.field_id.model_id != self.res_model_id for rule in self.rule_ids):
|
|
self.rule_ids = [(5, 0, 0)]
|
|
|
|
def _compute_records_to_clean(self):
|
|
count_data = self.env['data_cleaning.record']._read_group(
|
|
[('cleaning_model_id', 'in', self.ids)],
|
|
['cleaning_model_id'],
|
|
['__count'])
|
|
counts = {cleaning_model.id: count for cleaning_model, count in count_data}
|
|
for cm_model in self:
|
|
cm_model.records_to_clean_count = counts[cm_model.id] if cm_model.id in counts else 0
|
|
|
|
def _cron_clean_records(self):
|
|
self.sudo().search([])._clean_records(batch_commits=True)
|
|
self.sudo()._notify_records_to_clean()
|
|
|
|
def _clean_records_format_phone(self, actions, field):
|
|
self.ensure_one()
|
|
|
|
self._cr.execute("""
|
|
SELECT res_id, data_cleaning_rule_id
|
|
FROM data_cleaning_record
|
|
JOIN data_cleaning_record_data_cleaning_rule_rel
|
|
ON data_cleaning_record_data_cleaning_rule_rel.data_cleaning_record_id = data_cleaning_record.id""")
|
|
existing_rows = self._cr.fetchall()
|
|
|
|
records = self.env[self.res_model_name].search([(field, 'not in', [False, ''])])
|
|
records = records.with_context(prefetch_fields=False)
|
|
# Avoids multiple select queries when reading fields in _get_country_id and record[field].
|
|
records.read([fname for fname in ['country_id', 'company_id'] if fname in records] + [field])
|
|
field_id = actions[field]['field_id']
|
|
rule_ids = actions[field]['rule_ids']
|
|
result = []
|
|
for record in records:
|
|
record_country = self.env['data_cleaning.record']._get_country_id(record)
|
|
formatted = self.env[self.res_model_name]._phone_format(number=record[field], country=record_country, force_format='INTERNATIONAL')
|
|
if (record.id, rule_ids[0]) not in existing_rows and formatted and record[field] != formatted:
|
|
result.append({
|
|
'res_id': record['id'],
|
|
'rule_ids': rule_ids,
|
|
'cleaning_model_id': self.id,
|
|
'field_id': field_id,
|
|
})
|
|
return result
|
|
|
|
|
|
def _clean_records(self, batch_commits=False):
|
|
self.env.flush_all()
|
|
|
|
lang = self.env.user.lang
|
|
records_to_clean = []
|
|
for cleaning_model in self:
|
|
records_to_create = []
|
|
actions = cleaning_model.rule_ids._action_to_sql()
|
|
for field in actions:
|
|
action = actions[field]['action']
|
|
field_id = actions[field]['field_id']
|
|
rule_ids = actions[field]['rule_ids']
|
|
operator = actions[field]['operator']
|
|
cleaner = getattr(cleaning_model, '_clean_records_%s' % action, None)
|
|
if cleaner:
|
|
values = cleaner(actions, field)
|
|
records_to_create += values
|
|
else:
|
|
active_model = self.env[cleaning_model.res_model_name]
|
|
active_name = active_model._active_name
|
|
active_cond = sql.SQL("AND {}").format(sql.Identifier(active_name)) if active_name else sql.SQL('')
|
|
|
|
field_name = sql.Identifier(field)
|
|
cleaned_field_expr = sql.SQL(action.format(field))
|
|
|
|
if active_model._fields[field].translate:
|
|
field_name = sql.SQL("COALESCE({field}->>{lang}, {field}->>'en_US')").format(
|
|
field=sql.Identifier(field),
|
|
lang=sql.Literal(lang)
|
|
)
|
|
action = action.format("COALESCE({field}->>{lang}, {field}->>'en_US')")
|
|
cleaned_field_expr = sql.SQL(action).format(
|
|
field=sql.Identifier(field),
|
|
lang=sql.Literal(lang)
|
|
)
|
|
|
|
query = sql.SQL("""
|
|
SELECT
|
|
id AS res_id
|
|
FROM
|
|
{table}
|
|
WHERE
|
|
{field_name} {operator} {cleaned_field_expr}
|
|
AND NOT EXISTS(
|
|
SELECT 1
|
|
FROM {cleaning_record_table}
|
|
WHERE
|
|
res_id = {table}.id
|
|
AND cleaning_model_id = %s)
|
|
{active_cond}
|
|
ORDER BY id
|
|
""").format(
|
|
table=sql.Identifier(self.env[cleaning_model.res_model_name]._table),
|
|
field_name=field_name,
|
|
operator=sql.SQL(operator),
|
|
# can be complex sql expression & multiple actions get
|
|
# combined through string formatting, so doesn't seem
|
|
# to be a smarter solution than whitelisting the entire thing
|
|
cleaned_field_expr=cleaned_field_expr,
|
|
cleaning_record_table=sql.Identifier(self.env['data_cleaning.record']._table),
|
|
active_cond=active_cond
|
|
)
|
|
self._cr.execute(query, [cleaning_model.id])
|
|
for r in self._cr.fetchall():
|
|
records_to_create.append({
|
|
'res_id': r[0],
|
|
'rule_ids': rule_ids,
|
|
'cleaning_model_id': cleaning_model.id,
|
|
'field_id': field_id,
|
|
})
|
|
|
|
if cleaning_model.cleaning_mode == 'automatic':
|
|
for records_to_create_batch in split_every(DR_CREATE_STEP_AUTO, records_to_create):
|
|
self.env['data_cleaning.record'].create(records_to_create_batch).action_validate()
|
|
if batch_commits:
|
|
# Commit after each batch iteration to avoid complete rollback on timeout as
|
|
# this can create lots of new records.
|
|
self.env.cr.commit()
|
|
else:
|
|
records_to_clean = records_to_clean + records_to_create
|
|
for records_to_clean_batch in split_every(DR_CREATE_STEP_MANUAL, records_to_clean):
|
|
self.env['data_cleaning.record'].create(records_to_clean_batch)
|
|
if batch_commits:
|
|
self.env.cr.commit()
|
|
|
|
@api.model
|
|
def _notify_records_to_clean(self):
|
|
for cleaning_model in self.search([('cleaning_mode', '=', 'manual')]):
|
|
if not cleaning_model.notify_user_ids or not cleaning_model.notify_frequency:
|
|
continue
|
|
|
|
if cleaning_model.notify_frequency_period == 'days':
|
|
delta = relativedelta(days=cleaning_model.notify_frequency)
|
|
elif cleaning_model.notify_frequency_period == 'weeks':
|
|
delta = relativedelta(weeks=cleaning_model.notify_frequency)
|
|
else:
|
|
delta = relativedelta(months=cleaning_model.notify_frequency)
|
|
|
|
if not cleaning_model.last_notification or\
|
|
(cleaning_model.last_notification + delta) < fields.Datetime.now():
|
|
cleaning_model.last_notification = fields.Datetime.now()
|
|
cleaning_model._send_notification(delta)
|
|
|
|
def _send_notification(self, delta):
|
|
self.ensure_one()
|
|
last_date = fields.Date.today() - delta
|
|
records_count = self.env['data_cleaning.record'].search_count([
|
|
('cleaning_model_id', '=', self.id),
|
|
('create_date', '>=', last_date)
|
|
])
|
|
|
|
if records_count:
|
|
partner_ids = self.notify_user_ids.partner_id.ids
|
|
menu_id = self.env.ref('data_recycle.menu_data_cleaning_root').id
|
|
self.env['mail.thread'].message_notify(
|
|
body=self.env['ir.qweb']._render(
|
|
'data_cleaning.notification',
|
|
dict(
|
|
records_count=records_count,
|
|
res_model_label=self.res_model_id.name,
|
|
cleaning_model_id=self.id,
|
|
menu_id=menu_id
|
|
)
|
|
),
|
|
model=self._name,
|
|
notify_author=True,
|
|
partner_ids=partner_ids,
|
|
res_id=self.id,
|
|
subject=_('Data to Clean'),
|
|
)
|
|
|
|
############
|
|
# Overrides
|
|
############
|
|
def write(self, vals):
|
|
if 'active' in vals and not vals['active']:
|
|
self.env['data_cleaning.record'].search([('cleaning_model_id', 'in', self.ids)]).unlink()
|
|
return super().write(vals)
|
|
|
|
##########
|
|
# Actions
|
|
##########
|
|
def open_records(self):
|
|
self.ensure_one()
|
|
|
|
action = self.env["ir.actions.actions"]._for_xml_id("data_cleaning.action_data_cleaning_record")
|
|
action['context'] = dict(ast.literal_eval(action.get('context')), searchpanel_default_cleaning_model_id=self.id)
|
|
return action
|
|
|
|
def action_clean_records(self):
|
|
self.sudo()._clean_records()
|
|
|
|
if self.cleaning_mode == 'manual':
|
|
return self.open_records()
|