Closes #14438: Database representation of scripts

- Introduces the Script model to represent individual Python classes within a ScriptModule file - Automatically migrates jobs & event rules --------- Co-authored-by: Jeremy Stretch <jstretch@netboxlabs.com>

Author: arthanson

netbox/extras/api/serializers.py

@@ -44,9 +44,6 @@
  'ImageAttachmentSerializer',
  'JournalEntrySerializer',
  'ObjectChangeSerializer',
- 'ReportDetailSerializer',
- 'ReportSerializer',
- 'ReportInputSerializer',
  'SavedFilterSerializer',
  'ScriptDetailSerializer',
  'ScriptInputSerializer',
@@ -85,9 +82,9 @@ def get_action_object(self, instance):
  context = {'request': self.context['request']}
  # We need to manually instantiate the serializer for scripts
  if instance.action_type == EventRuleActionChoices.SCRIPT:
- script_name = instance.action_parameters['script_name']
- script = instance.action_object.scripts[script_name]()
- return NestedScriptSerializer(script, context=context).data
+ script = instance.action_object
+ instance = script.python_class() if script.python_class else None
+ return NestedScriptSerializer(instance, context=context).data
  else:
  serializer = get_serializer_for_model(
  model=instance.action_object_type.model_class(),
@@ -512,79 +509,54 @@ class Meta:
  ]
 
 
-#
-# Reports
-#
-
-class ReportSerializer(serializers.Serializer):
- url = serializers.HyperlinkedIdentityField(
- view_name='extras-api:report-detail',
- lookup_field='full_name',
- lookup_url_kwarg='pk'
- )
- id = serializers.CharField(read_only=True, source="full_name")
- module = serializers.CharField(max_length=255)
- name = serializers.CharField(max_length=255)
- description = serializers.CharField(max_length=255, required=False)
- test_methods = serializers.ListField(child=serializers.CharField(max_length=255), read_only=True)
- result = NestedJobSerializer()
- display = serializers.SerializerMethodField(read_only=True)
-
- @extend_schema_field(serializers.CharField())
- def get_display(self, obj):
- return f'{obj.name} ({obj.module})'
-
-
-class ReportDetailSerializer(ReportSerializer):
- result = JobSerializer()
-
-
-class ReportInputSerializer(serializers.Serializer):
- schedule_at = serializers.DateTimeField(required=False, allow_null=True)
- interval = serializers.IntegerField(required=False, allow_null=True)
-
- def validate_schedule_at(self, value):
- if value and not self.context['report'].scheduling_enabled:
- raise serializers.ValidationError(_("Scheduling is not enabled for this report."))
- return value
-
- def validate_interval(self, value):
- if value and not self.context['report'].scheduling_enabled:
- raise serializers.ValidationError(_("Scheduling is not enabled for this report."))
- return value
-
-
 #
 # Scripts
 #
 
-class ScriptSerializer(serializers.Serializer):
- url = serializers.HyperlinkedIdentityField(
- view_name='extras-api:script-detail',
- lookup_field='full_name',
- lookup_url_kwarg='pk'
- )
- id = serializers.CharField(read_only=True, source="full_name")
- module = serializers.CharField(max_length=255)
- name = serializers.CharField(read_only=True)
- description = serializers.CharField(read_only=True)
+class ScriptSerializer(ValidatedModelSerializer):
+ url = serializers.HyperlinkedIdentityField(view_name='extras-api:script-detail')
+ description = serializers.SerializerMethodField(read_only=True)
  vars = serializers.SerializerMethodField(read_only=True)
- result = NestedJobSerializer()
- display = serializers.SerializerMethodField(read_only=True)
+ result = NestedJobSerializer(read_only=True)
+
+ class Meta:
+ model = Script
+ fields = [
+ 'id', 'url', 'module', 'name', 'description', 'vars', 'result', 'display', 'is_executable',
+ ]
 
  @extend_schema_field(serializers.JSONField(allow_null=True))
- def get_vars(self, instance):
- return {
- k: v.__class__.__name__ for k, v in instance._get_vars().items()
- }
+ def get_vars(self, obj):
+ if obj.python_class:
+ return {
+ k: v.__class__.__name__ for k, v in obj.python_class()._get_vars().items()
+ }
+ else:
+ return {}
 
  @extend_schema_field(serializers.CharField())
  def get_display(self, obj):
  return f'{obj.name} ({obj.module})'
 
+ @extend_schema_field(serializers.CharField())
+ def get_description(self, obj):
+ if obj.python_class:
+ return obj.python_class().description
+ else:
+ return None
+
 
 class ScriptDetailSerializer(ScriptSerializer):
- result = JobSerializer()
+ result = serializers.SerializerMethodField(read_only=True)
+
+ @extend_schema_field(JobSerializer())
+ def get_result(self, obj):
+ job = obj.jobs.all().order_by('-created').first()
+ context = {
+ 'request': self.context['request']
+ }
+ data = JobSerializer(job, context=context).data
+ return data
 
 
 class ScriptInputSerializer(serializers.Serializer):

netbox/extras/api/views.py

@@ -1,5 +1,4 @@
 from django.contrib.contenttypes.models import ContentType
-from django.http import Http404
 from django.shortcuts import get_object_or_404
 from django_rq.queues import get_connection
 from rest_framework import status
@@ -9,14 +8,13 @@
 from rest_framework.renderers import JSONRenderer
 from rest_framework.response import Response
 from rest_framework.routers import APIRootView
-from rest_framework.viewsets import ReadOnlyModelViewSet, ViewSet
+from rest_framework.viewsets import ModelViewSet, ReadOnlyModelViewSet
 from rq import Worker
 
-from core.choices import JobStatusChoices
 from core.models import Job
 from extras import filtersets
 from extras.models import *
-from extras.scripts import get_module_and_script, run_script
+from extras.scripts import run_script
 from netbox.api.authentication import IsAuthenticatedOrLoginNotRequired
 from netbox.api.features import SyncedDataMixin
 from netbox.api.metadata import ContentTypeMetadata
@@ -209,66 +207,30 @@ def render(self, request, pk):
 # Scripts
 #
 
-class ScriptViewSet(ViewSet):
+class ScriptViewSet(ModelViewSet):
  permission_classes = [IsAuthenticatedOrLoginNotRequired]
+ queryset = Script.objects.prefetch_related('jobs')
+ serializer_class = serializers.ScriptSerializer
+ filterset_class = filtersets.ScriptFilterSet
+
  _ignore_model_permissions = True
- schema = None
  lookup_value_regex = '[^/]+' # Allow dots
 
- def _get_script(self, pk):
- try:
- module_name, script_name = pk.split('.', maxsplit=1)
- except ValueError:
- raise Http404
-
- module, script = get_module_and_script(module_name, script_name)
- if script is None:
- raise Http404
-
- return module, script
-
- def list(self, request):
- results = {
- job.name: job
- for job in Job.objects.filter(
- object_type=ContentType.objects.get(app_label='extras', model='scriptmodule'),
- status__in=JobStatusChoices.TERMINAL_STATE_CHOICES
- ).order_by('name', '-created').distinct('name').defer('data')
- }
-
- script_list = []
- for script_module in ScriptModule.objects.restrict(request.user):
- script_list.extend(script_module.scripts.values())
-
- # Attach Job objects to each script (if any)
- for script in script_list:
- script.result = results.get(script.class_name, None)
-
- serializer = serializers.ScriptSerializer(script_list, many=True, context={'request': request})
-
- return Response({'count': len(script_list), 'results': serializer.data})
-
  def retrieve(self, request, pk):
- module, script = self._get_script(pk)
- object_type = ContentType.objects.get(app_label='extras', model='scriptmodule')
- script.result = Job.objects.filter(
- object_type=object_type,
- name=script.class_name,
- status__in=JobStatusChoices.TERMINAL_STATE_CHOICES
- ).first()
+ script = get_object_or_404(self.queryset, pk=pk)
  serializer = serializers.ScriptDetailSerializer(script, context={'request': request})
 
  return Response(serializer.data)
 
  def post(self, request, pk):
  """
- Run a Script identified as "<module>.<script>" and return the pending Job as the result
+ Run a Script identified by the id and return the pending Job as the result
  """
 
  if not request.user.has_perm('extras.run_script'):
  raise PermissionDenied("This user does not have permission to run scripts.")
 
- module, script = self._get_script(pk)
+ script = get_object_or_404(self.queryset, pk=pk)
  input_serializer = serializers.ScriptInputSerializer(
  data=request.data,
  context={'script': script}
@@ -281,13 +243,13 @@ def post(self, request, pk):
  if input_serializer.is_valid():
  script.result = Job.enqueue(
  run_script,
- instance=module,
- name=script.class_name,
+ instance=script.module,
+ name=script.python_class.class_name,
  user=request.user,
  data=input_serializer.data['data'],
  request=copy_safe_request(request),
  commit=input_serializer.data['commit'],
- job_timeout=script.job_timeout,
+ job_timeout=script.python_class.job_timeout,
  schedule_at=input_serializer.validated_data.get('schedule_at'),
  interval=input_serializer.validated_data.get('interval')
  )

netbox/extras/events.py

@@ -116,15 +116,13 @@ def process_event_rules(event_rules, model_name, event, data, username=None, sna
  # Scripts
  elif event_rule.action_type == EventRuleActionChoices.SCRIPT:
  # Resolve the script from action parameters
- script_module = event_rule.action_object
- script_name = event_rule.action_parameters['script_name']
- script = script_module.scripts[script_name]()
+ script = event_rule.action_object.python_class()
 
  # Enqueue a Job to record the script's execution
  Job.enqueue(
  "extras.scripts.run_script",
- instance=script_module,
- name=script.class_name,
+ instance=script.module,
+ name=script.name,
  user=user,
  data=data
  )

netbox/extras/filtersets.py

@@ -29,11 +29,32 @@
  'LocalConfigContextFilterSet',
  'ObjectChangeFilterSet',
  'SavedFilterFilterSet',
+ 'ScriptFilterSet',
  'TagFilterSet',
  'WebhookFilterSet',
 )
 
 
+class ScriptFilterSet(BaseFilterSet):
+ q = django_filters.CharFilter(
+ method='search',
+ label=_('Search'),
+ )
+
+ class Meta:
+ model = Script
+ fields = [
+ 'id', 'name',
+ ]
+
+ def search(self, queryset, name, value):
+ if not value.strip():
+ return queryset
+ return queryset.filter(
+ Q(name__icontains=value)
+ )
+
+
 class WebhookFilterSet(NetBoxModelFilterSet):
  q = django_filters.CharFilter(
  method='search',

netbox/extras/forms/bulk_import.py

@@ -212,11 +212,8 @@ def clean(self):
  module, script = get_module_and_script(module_name, script_name)
  except ObjectDoesNotExist:
  raise forms.ValidationError(_("Script {name} not found").format(name=action_object))
- self.instance.action_object = module
- self.instance.action_object_type = ContentType.objects.get_for_model(module, for_concrete_model=False)
- self.instance.action_parameters = {
- 'script_name': script_name,
- }
+ self.instance.action_object = script
+ self.instance.action_object_type = ContentType.objects.get_for_model(script, for_concrete_model=False)
 
 
 class TagImportForm(CSVModelForm):

netbox/extras/forms/model_forms.py

@@ -297,20 +297,16 @@ class Meta:
  }
 
  def init_script_choice(self):
- choices = []
- for module in ScriptModule.objects.all():
- scripts = []
- for script_name in module.scripts.keys():
- name = f"{str(module.pk)}:{script_name}"
- scripts.append((name, script_name))
- if scripts:
- choices.append((str(module), scripts))
- self.fields['action_choice'].choices = choices
-
- if self.instance.action_type == EventRuleActionChoices.SCRIPT and self.instance.action_parameters:
- scriptmodule_id = self.instance.action_object_id
- script_name = self.instance.action_parameters.get('script_name')
- self.fields['action_choice'].initial = f'{scriptmodule_id}:{script_name}'
+ initial = None
+ if self.instance.action_type == EventRuleActionChoices.SCRIPT:
+ script_id = get_field_value(self, 'action_object_id')
+ initial = Script.objects.get(pk=script_id) if script_id else None
+ self.fields['action_choice'] = DynamicModelChoiceField(
+ label=_('Script'),
+ queryset=Script.objects.all(),
+ required=True,
+ initial=initial
+ )
 
  def init_webhook_choice(self):
  initial = None
@@ -348,26 +344,13 @@ def clean(self):
  # Script
  elif self.cleaned_data.get('action_type') == EventRuleActionChoices.SCRIPT:
  self.cleaned_data['action_object_type'] = ContentType.objects.get_for_model(
- ScriptModule,
+ Script,
  for_concrete_model=False
  )
- module_id, script_name = action_choice.split(":", maxsplit=1)
- self.cleaned_data['action_object_id'] = module_id
+ self.cleaned_data['action_object_id'] = action_choice.id
 
  return self.cleaned_data
 
- def save(self, *args, **kwargs):
- # Set action_parameters on the instance
- if self.cleaned_data['action_type'] == EventRuleActionChoices.SCRIPT:
- module_id, script_name = self.cleaned_data.get('action_choice').split(":", maxsplit=1)
- self.instance.action_parameters = {
- 'script_name': script_name,
- }
- else:
- self.instance.action_parameters = None
-
- return super().save(*args, **kwargs)
-
 
 class TagForm(forms.ModelForm):
  slug = SlugField()