Browse Source

Merge branch 'release/0.3.0'

master
Sam Black 4 years ago
parent
commit
d122a4bae2
Signed by: samwwwblack GPG Key ID: 0FF0223994EA47D8
  1. 3
      .gitignore
  2. 56
      CHANGELOG.rst
  3. 24
      README.rst
  4. 24
      Vagrantfile
  5. 6
      babel/babel.cfg
  6. 8
      cupola.yaml
  7. 4
      cupola/__init__.py
  8. 4
      cupola/api/run.py
  9. 8
      cupola/api/templates/api/event.html
  10. 6
      cupola/api/templates/api/event.txt
  11. 24
      cupola/api/templates/api/event_limit.html
  12. 15
      cupola/api/templates/api/event_limit.txt
  13. 574
      cupola/api/utils.py
  14. 380
      cupola/api/view.py
  15. 2
      cupola/config.py
  16. 42
      cupola/forms/projects.py
  17. 18
      cupola/forms/users.py
  18. 75
      cupola/forms/utils.py
  19. 7
      cupola/models/__init__.py
  20. 51
      cupola/models/events.py
  21. 80
      cupola/models/projects.py
  22. 14
      cupola/models/users.py
  23. 11
      cupola/static/css/cupola.css
  24. 7935
      cupola/static/css/patternfly-additions.css
  25. 10519
      cupola/static/css/patternfly.css
  26. BIN
      cupola/static/fonts/FontAwesome.otf
  27. BIN
      cupola/static/fonts/OpenSans-Bold-webfont.eot
  28. 20852
      cupola/static/fonts/OpenSans-Bold-webfont.svg
  29. BIN
      cupola/static/fonts/OpenSans-Bold-webfont.ttf
  30. BIN
      cupola/static/fonts/OpenSans-Bold-webfont.woff
  31. BIN
      cupola/static/fonts/OpenSans-Bold-webfont.woff2
  32. BIN
      cupola/static/fonts/OpenSans-BoldItalic-webfont.eot
  33. 20860
      cupola/static/fonts/OpenSans-BoldItalic-webfont.svg
  34. BIN
      cupola/static/fonts/OpenSans-BoldItalic-webfont.ttf
  35. BIN
      cupola/static/fonts/OpenSans-BoldItalic-webfont.woff
  36. BIN
      cupola/static/fonts/OpenSans-BoldItalic-webfont.woff2
  37. BIN
      cupola/static/fonts/OpenSans-ExtraBold-webfont.eot
  38. 20854
      cupola/static/fonts/OpenSans-ExtraBold-webfont.svg
  39. BIN
      cupola/static/fonts/OpenSans-ExtraBold-webfont.ttf
  40. BIN
      cupola/static/fonts/OpenSans-ExtraBold-webfont.woff
  41. BIN
      cupola/static/fonts/OpenSans-ExtraBold-webfont.woff2
  42. BIN
      cupola/static/fonts/OpenSans-ExtraBoldItalic-webfont.eot
  43. 20860
      cupola/static/fonts/OpenSans-ExtraBoldItalic-webfont.svg
  44. BIN
      cupola/static/fonts/OpenSans-ExtraBoldItalic-webfont.ttf
  45. BIN
      cupola/static/fonts/OpenSans-ExtraBoldItalic-webfont.woff
  46. BIN
      cupola/static/fonts/OpenSans-ExtraBoldItalic-webfont.woff2
  47. BIN
      cupola/static/fonts/OpenSans-Italic-webfont.eot
  48. 20867
      cupola/static/fonts/OpenSans-Italic-webfont.svg
  49. BIN
      cupola/static/fonts/OpenSans-Italic-webfont.ttf
  50. BIN
      cupola/static/fonts/OpenSans-Italic-webfont.woff
  51. BIN
      cupola/static/fonts/OpenSans-Italic-webfont.woff2
  52. BIN
      cupola/static/fonts/OpenSans-Light-webfont.eot
  53. 20851
      cupola/static/fonts/OpenSans-Light-webfont.svg
  54. BIN
      cupola/static/fonts/OpenSans-Light-webfont.ttf
  55. BIN
      cupola/static/fonts/OpenSans-Light-webfont.woff
  56. BIN
      cupola/static/fonts/OpenSans-Light-webfont.woff2
  57. BIN
      cupola/static/fonts/OpenSans-LightItalic-webfont.eot
  58. 20868
      cupola/static/fonts/OpenSans-LightItalic-webfont.svg
  59. BIN
      cupola/static/fonts/OpenSans-LightItalic-webfont.ttf
  60. BIN
      cupola/static/fonts/OpenSans-LightItalic-webfont.woff
  61. BIN
      cupola/static/fonts/OpenSans-LightItalic-webfont.woff2
  62. BIN
      cupola/static/fonts/OpenSans-Regular-webfont.eot
  63. 20855
      cupola/static/fonts/OpenSans-Regular-webfont.svg
  64. BIN
      cupola/static/fonts/OpenSans-Regular-webfont.ttf
  65. BIN
      cupola/static/fonts/OpenSans-Regular-webfont.woff
  66. BIN
      cupola/static/fonts/OpenSans-Regular-webfont.woff2
  67. BIN
      cupola/static/fonts/OpenSans-Semibold-webfont.eot
  68. 20854
      cupola/static/fonts/OpenSans-Semibold-webfont.svg
  69. BIN
      cupola/static/fonts/OpenSans-Semibold-webfont.ttf
  70. BIN
      cupola/static/fonts/OpenSans-Semibold-webfont.woff
  71. BIN
      cupola/static/fonts/OpenSans-Semibold-webfont.woff2
  72. BIN
      cupola/static/fonts/OpenSans-SemiboldItalic-webfont.eot
  73. 20867
      cupola/static/fonts/OpenSans-SemiboldItalic-webfont.svg
  74. BIN
      cupola/static/fonts/OpenSans-SemiboldItalic-webfont.ttf
  75. BIN
      cupola/static/fonts/OpenSans-SemiboldItalic-webfont.woff
  76. BIN
      cupola/static/fonts/OpenSans-SemiboldItalic-webfont.woff2
  77. BIN
      cupola/static/fonts/PatternFlyIcons-webfont.eot
  78. 142
      cupola/static/fonts/PatternFlyIcons-webfont.svg
  79. BIN
      cupola/static/fonts/PatternFlyIcons-webfont.ttf
  80. BIN
      cupola/static/fonts/PatternFlyIcons-webfont.woff
  81. BIN
      cupola/static/fonts/fontawesome-webfont.eot
  82. 3230
      cupola/static/fonts/fontawesome-webfont.svg
  83. BIN
      cupola/static/fonts/fontawesome-webfont.ttf
  84. BIN
      cupola/static/fonts/fontawesome-webfont.woff
  85. BIN
      cupola/static/fonts/fontawesome-webfont.woff2
  86. BIN
      cupola/static/fonts/glyphicons-halflings-regular.eot
  87. 288
      cupola/static/fonts/glyphicons-halflings-regular.svg
  88. BIN
      cupola/static/fonts/glyphicons-halflings-regular.ttf
  89. BIN
      cupola/static/fonts/glyphicons-halflings-regular.woff
  90. BIN
      cupola/static/fonts/glyphicons-halflings-regular.woff2
  91. 0
      cupola/static/img/apple-touch-icon-precomposed-114.png
  92. 0
      cupola/static/img/apple-touch-icon-precomposed-144.png
  93. 0
      cupola/static/img/apple-touch-icon-precomposed-57.png
  94. 0
      cupola/static/img/apple-touch-icon-precomposed-72.png
  95. 0
      cupola/static/img/bg-login.jpg
  96. BIN
      cupola/static/img/brand-lg.png
  97. BIN
      cupola/static/img/brand.png
  98. 87
      cupola/static/img/brand.svg
  99. 0
      cupola/static/img/logo.png
  100. 0
      cupola/static/img/logo.svg

3
.gitignore

@ -87,6 +87,9 @@ com_crashlytics_export_strings.xml
crashlytics.properties
crashlytics-build.properties
# SonarQube
.scannerwork/
sonar-project.properties
# Vagrant
.vagrant/

56
CHANGELOG.rst

@ -5,16 +5,38 @@ Cupola ChangeLog
All notable changes to this project will be documented in this file.
This project adheres to `PEP 440 <https://www.python.org/dev/peps/pep-0440/>`_.
Unreleased_
===========
0.3.0_ - 2017-11-15
===================
.. _0.3.0: https://code.lapwing.org/devops/cupola/tree/0.3.0
Added
-----
- Per user, per project notification settings
- Interface translation support
- Public DSN only (for example Javascript) Sentry client support
- Show "breadcrumb" data
Changed
-------
- Update to work with latest Flask
- Update to latest PatternFly
- Update to work with latest Sentry clients
.. _Unreleased: https://github.com/LapwingOrg/cupola/compare/0.2.0...develop
Removed
-------
- Vagrant support
0.2.0_ - 2016-02-11
===================
.. _0.2.0: https://github.com/LapwingOrg/cupola/compare/0.1.10...0.2.0
.. _0.2.0: https://code.lapwing.org/devops/cupola/tree/0.2.0
Added
-----
@ -39,7 +61,7 @@ Fixed
0.1.10_ - 2015-11-02
====================
.. _0.1.10: https://github.com/LapwingOrg/cupola/compare/0.1.9...0.1.10
.. _0.1.10: https://code.lapwing.org/devops/cupola/tree/0.1.10
Fixed
-----
@ -50,7 +72,7 @@ Fixed
0.1.9_ - 2015-11-01
===================
.. _0.1.9: https://github.com/LapwingOrg/cupola/compare/0.1.8...0.1.9
.. _0.1.9: https://code.lapwing.org/devops/cupola/tree/0.1.9
Fixed
-----
@ -63,7 +85,7 @@ Fixed
0.1.8_ - 2015-10-07
===================
.. _0.1.8: https://github.com/LapwingOrg/cupola/compare/0.1.7...0.1.8
.. _0.1.8: https://code.lapwing.org/devops/cupola/tree/0.1.8
Changed
-------
@ -74,7 +96,7 @@ Changed
0.1.7_ - 2015-08-26
===================
.. _0.1.7: https://github.com/LapwingOrg/cupola/compare/0.1.6...0.1.7
.. _0.1.7: https://code.lapwing.org/devops/cupola/tree/0.1.7
Fixed
-----
@ -86,7 +108,7 @@ Fixed
0.1.6_ - 2015-08-26
===================
.. _0.1.6: https://github.com/LapwingOrg/cupola/compare/0.1.5...0.1.6
.. _0.1.6: https://code.lapwing.org/devops/cupola/tree/0.1.6
Added
-----
@ -98,7 +120,7 @@ Added
0.1.5_ - 2015-08-24
===================
.. _0.1.5: https://github.com/LapwingOrg/cupola/compare/0.1.4...0.1.5
.. _0.1.5: https://code.lapwing.org/devops/cupola/tree/0.1.5
Fixed
-----
@ -109,7 +131,7 @@ Fixed
0.1.4_ - 2015-08-24
===================
.. _0.1.4: https://github.com/LapwingOrg/cupola/compare/0.1.3...0.1.4
.. _0.1.4: https://code.lapwing.org/devops/cupola/tree/0.1.4
Changed
-------
@ -126,7 +148,7 @@ Fixed
0.1.3_ - 2015-08-24
===================
.. _0.1.3: https://github.com/LapwingOrg/cupola/compare/0.1.2...0.1.3
.. _0.1.3: https://code.lapwing.org/devops/cupola/tree/0.1.3
Added
-----
@ -143,7 +165,7 @@ Fixed
0.1.2_ - 2015-08-24
===================
.. _0.1.2: https://github.com/LapwingOrg/cupola/compare/0.1.1...0.1.2
.. _0.1.2: https://code.lapwing.org/devops/cupola/tree/0.1.2
Fixed
-----
@ -154,7 +176,7 @@ Fixed
0.1.1_ - 2015-08-24
===================
.. _0.1.1: https://github.com/LapwingOrg/cupola/compare/0.1...0.1.1
.. _0.1.1: https://code.lapwing.org/devops/cupola/tree/0.1.1
Added
-----
@ -163,8 +185,10 @@ Added
- Separate API daemon
0.1 - 2015-08-24
================
0.1_ - 2015-08-24
=================
.. _0.1: https://code.lapwing.org/devops/cupola/tree/0.1
Added
-----

24
README.rst

@ -18,14 +18,15 @@ Requirements
Software
--------
- Python 3 (3.4 or greater)
- PostgreSQL (9.4 or greater)
- Python 3 (3.5 or greater)
- PostgreSQL (9.5 or greater)
- Redis (2.6 or greater)
Hardware
--------
- 512MB RAM (1GB+ recommended)
- 512MB RAM
- 1CPU core
- Storage requirement will depend on the number of projects,
number and frequecy of events
@ -33,21 +34,6 @@ Hardware
5GB or so will likely be plenty for most use cases.
Setup
=====
Vagrant
-------
#. ``vagrant up``
- If you are editing the Vagrant file, use ``DNF_PROXY="proxy" vagrant up`` instead,
with a suitable caching proxy running on ``192.168.121.1:3128``
#. Goto ``http://127.0.0.1:8008`` and login.
- Default login: ``admin@cupola.example.com``:``password``
ChangeLog
=========
@ -65,7 +51,7 @@ TODO (probably)
Licence
=======
Cupola is Copyright (C) 2015-2016 Sam Black samwwwblack@lapwing.org.
Cupola is Copyright (C) 2015-2017 Sam Black samwwwblack@lapwing.org.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by

24
Vagrantfile

@ -1,24 +0,0 @@
# -*- mode: ruby -*-
# vi: set ft=ruby :
Vagrant.configure(2) do |config|
config.vm.box = "fedora/23-cloud-base"
config.vm.network "forwarded_port", guest: 8008, host: 8008
config.vm.provider "libvirt" do |libvirt|
libvirt.driver = "kvm"
libvirt.memory = 1024
libvirt.cpus = 1
end
config.vm.provision :shell, path: "deployment/vagrant/bootstrap.sh", args: ENV["DNF_PROXY"]
config.vm.synced_folder ".", "/vagrant", type: "rsync", rsync__exclude: ".git/"
config.vm.synced_folder "./cupola", "/home/cupola/cupola/cupola",
type: "rsync", owner: "cupola", group: "nginx",
rsync__args: ["--verbose", "--rsync-path='sudo rsync'",
"--chown=cupola:nginx",
"--archive", "--delete", "-z"]
end

6
babel/babel.cfg

@ -0,0 +1,6 @@
[python: cupola/**.py]
[python: run.py]
[jinja2: cupola/**/templates/**.html]
extensions=jinja2.ext.autoescape,jinja2.ext.with_
[jinja2: cupola/**/templates/**.txt]
extensions=jinja2.ext.autoescape,jinja2.ext.with_

8
cupola.yaml

@ -1,7 +1,9 @@
image:
name: lapwing.org/fedora23/cupola
version: 0.2.0
srcimage: lapwing.org/fedora23/flask
name: lapwing.org/fedora25/cupola
version: 0.3.0
srcimage:
name: lapwing.org/fedora25/flask
version: v20170809
build:
- copy:
src: ../cupola

4
cupola/__init__.py

@ -1,7 +1,7 @@
# coding=utf8
#
# config.py: Basic configuration
# Copyright (C) 2015-2016 Sam Black <samwwwblack@lapwing.org>
# Copyright (C) 2015-2017 Sam Black <samwwwblack@lapwing.org>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
@ -17,4 +17,4 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
__author__ = "Sam Black <samwwwblack@lapwing.org>"
__version__ = "0.2.0"
__version__ = "0.3.0"

4
cupola/api/run.py

@ -20,7 +20,9 @@ from flask import Flask
from flask_migrate import Migrate
from cupola.models import db
from cupola.utils import babel
from cupola.utils import mail
from cupola.api.view import api
from cupola.api.queue import rq_manager
@ -44,6 +46,8 @@ def create_api():
db.init_app(app)
Migrate(app, db)
babel.init_app(app)
rq_manager.init_app(app)
mail.init_app(app)

8
cupola/api/templates/api/event.html

@ -1,9 +1,11 @@
Hi {{ user.username }},
{% trans username=user.username, project_name=group.project.name, group_message=group.message, event_url=event_url %}
Hi {{ username }},
<p>
Project <strong>{{ group.project.name }}</strong> has registered a new event
for event group <em>{{ group.message }}</em>.
Project <strong>{{ project_name }}</strong> has registered a new event
for event group <em>{{ group_message }}</em>.
</p>
<p>
You can view this event <a href="{{ event_url }}">here</a>.
</p>
{% endtrans %}

6
cupola/api/templates/api/event.txt

@ -1,5 +1,7 @@
Hi {{ user.username }},
{% trans username=user.username, project_name=group.project.name, group_message=group.message, event_url=event_url %}
Hi {{ username }},
Project "{{ group.project.name }}" has registered a new event for event group "{{ group.message }}".
Project "{{ project_name }}" has registered a new event for event group "{{ group_message }}".
You can view this event at {{ event_url }}
{% endtrans %}

24
cupola/api/templates/api/event_limit.html

@ -0,0 +1,24 @@
{% trans username=user.username, project_name=group.project.name, group_message=group.message %}
Hi {{ username }},
<p>
Project <strong>{{ project_name }}</strong> has registered a new event
for event group <em>{{ group_message }}</em>.
</p>
{% endtrans %}
{% trans event_count=event_count %}
<p>
There is an additional event that have been reported.
</p>
{% pluralize event_count %}
<p>
There are additional {{ event_count }} events that have been reported.
</p>
{% endtrans %}
{% trans event_list_url=event_list_url %}
<p>
You can view this list <a href="{{ event_list_url }}">here</a>.
</p>
{% endtrans %}

15
cupola/api/templates/api/event_limit.txt

@ -0,0 +1,15 @@
{% trans username=user.username, project_name=group.project.name, group_message=group.message %}
Hi {{ username }},
Project "{{ project_name }}" has registered a new event for event group "{{ group_message }}".
{% endtrans %}
{% trans event_count=event_count %}
There is an additional event that have been reported.
{% pluralize event_count %}
There are additional {{ event_count }} events that have been reported.
{% endtrans %}
{% trans event_list_url=event_list_url %}
You can view this list at {{ event_list_url }}
{% endtrans %}

574
cupola/api/utils.py

@ -0,0 +1,574 @@
# coding=utf8
#
# utils.py: cupola API utilities
# Copyright (C) 2017 Sam Black <samwwwblack@lapwing.org>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import arrow
import gzip
import json
import re
import time
import zlib
from datetime import datetime
from flask import Response
from flask import abort
from flask import render_template
from flask import url_for
from flask.json import jsonify
from flask_babelex import gettext
from flask_mail import Message
from cupola.api.queue import rq_manager
from cupola.models import db
from cupola.models.events import DEFAULT_LOG_LEVEL
from cupola.models.events import Breadcrumb
from cupola.models.events import Event
from cupola.models.events import EventGroup
from cupola.models.events import ExceptionModel
from cupola.models.events import LogLevels
from cupola.models.events import RequestModel
from cupola.models.events import Stacktrace
from cupola.models.events import StacktraceFrames
from cupola.models.events import Template
from cupola.models.events import UserModel
from cupola.models.projects import Project
from cupola.utils import mail
SENTRY_HEADER_AUTH = "X-Sentry-Auth"
SENTRY_HEADER_ERROR = "X-Sentry-Error"
def _cors_control_allow_origin(resp, origin=None):
"""
Add CORS header to allow cross origin calls.
:param resp: the response object to modify
:type resp: flask.Response
:param origin: the origin to allow, or None for all
:type origin: str or None
"""
if origin:
resp.headers.add("Access-Control-Allow-Origin", origin)
else:
resp.headers.add("Access-Control-Allow-Origin", "*")
class ApiBaseError(Response):
STATUS_CODE = 400
MESSAGE = "Invalid request"
def __init__(self, msg=None, origin=None):
if not msg:
msg = self.MESSAGE
super().__init__(msg, self.STATUS_CODE, [(SENTRY_HEADER_ERROR, msg)])
if origin:
_cors_control_allow_origin(self, origin)
class ApiAuthError(ApiBaseError):
STATUS_CODE = 401
MESSAGE = "Invalid credentials"
class ApiProcessError(ApiBaseError):
pass
class DecompressException(Exception):
pass
def _data_decode(data):
"""
Decode the data from bytes to string.
:param data: data to decode
:type data: bytes
:return: decoded string
:rtype: str
"""
return bytes.decode(data)
def _data_gzip(data):
"""
Decompress the gzipped data and decode from bytes to string.
:param data: gzip'd data to decode
:type data: gzip.GzipFile
:return: decoded string
:rtype: str
"""
try:
raw = gzip.decompress(data)
except OSError as e:
raise DecompressException()
else:
return _data_decode(raw)
def _data_decompress(data):
"""
Decompress the zlib'd data and decode bytes to string.
:param data: zlib'd data to decode
:type data: bytes
:return: decoded string
:rtype: str
"""
try:
raw = zlib.decompress(data)
except zlib.error:
raise DecompressException()
else:
return _data_decode(raw)
def get_payload(data, encoding=None, origin=None):
"""
"Store" the data
:param data: request data
:type data: str, bytes or gzip.GzipFile
:param encoding: the incoming encoding of the data,
should be one of `gzip`, `deflate` or None if unknown
:type encoding: str or None
:param origin: the origin URL of the request,
can be None if missing
:type origin: str or None
:return: JSON data loaded into dict
:rtype: dict
"""
if encoding and encoding in ("gzip", "decompress"):
if encoding == "gzip":
f = _data_gzip
elif encoding == "decompress":
f = _data_decompress
try:
txt = f(data)
except DecompressException:
return abort(ApiProcessError("Bad data decoding request", origin))
else:
# If the encoding is None, it could be that
# 1. There is no encoding
# 2. The encoding hasn't been detected
# We first try just loading the data into JSON,
# and if failed, then try decompressing it
try:
txt = _data_decode(data)
except UnicodeDecodeError:
pass
else:
try:
event_data = json.loads(txt)
except json.JSONDecodeError:
pass
else:
return event_data
for f in (_data_gzip, _data_decompress):
try:
txt = f(data)
except DecompressException:
continue
else:
break
else:
return abort(ApiProcessError("Bad data decoding request", origin))
if txt:
matches = set(re.findall("\"?'?([\w*\.]+\.\w*)'?\"?: ", txt))
for match in matches:
r = match.replace('.', u'\uff0e')
txt = txt.replace(match, r)
return json.loads(txt)
return {}
def payload_event_id(payload, origin=None):
"""
Create a JSON response with the event ID of the just registered event.
:param payload: payload of the event
:type payload: dict
:param origin: if the client is using a public DSN, the origin is set,
and requires setting CORS headers.
Can be None if using a normal DSN.
:type origin: str or None
:return: JSON response with appropriate headers
:rtype: flask.Response
"""
event_resp = jsonify({"id": payload.get("event_id")})
if origin:
_cors_control_allow_origin(event_resp, origin)
return event_resp
def _process_stacktrace_frames(stacktraces):
"""
Process and put in order stack trace frames.
:param stacktraces: list of stacktrace data frames
:type stacktraces: list
:return: list of StacktraceFrames
:rtype: list
"""
frames = []
for order, frame in enumerate(stacktraces):
# Stacktrace frames need to be in order to be meaningful
frame["order"] = order
new_frame = StacktraceFrames(**frame)
frames.append(new_frame)
return frames
def record_data(project_id, data, sentry_client):
"""
Record sentry data.
:param project_id: Project ID to dump data against.
:type project_id: int
:param data: data sent from the client
:type data: dict
:param sentry_client: the Sentry client version submitted,
can be None if missing
:type sentry_client: str or None
:return: event ID
:rtype: int
"""
from cupola.api.run import create_api
if project_id != data.pop("project"):
return
with create_api().app_context():
project = Project.query.get(project_id)
if not project:
return
# We do this processing after checking if the project exists
# to possibly save on CPU cycles
for sim in ("sentry.interfaces.Message",
u"sentry\uff0einterfaces\uff0eMessage"):
if sim in data:
# Document field is logentry
data["logentry"] = data[sim]
del data[sim]
break
if not data.get("message") and data.get("logentry"):
if data["logentry"].get("formatted"):
data["message"] = data["logentry"]["formatted"]
else:
data["message"] = data["logentry"]["message"]
if data.get("level"):
level = data["level"]
if (level and isinstance(level, str) and
hasattr(LogLevels, level.lower())):
data["level"] = getattr(LogLevels, level.lower())
elif not level:
data["level"] = DEFAULT_LOG_LEVEL
else:
data["level"] = DEFAULT_LOG_LEVEL
if not data.get("logger"):
data["logger"] = "<missing>"
exceptions = []
if data.get("exception"):
for exception_data in data["exception"]["values"]:
exception_stacktrace = exception_data.get("stacktrace")
exception_stacktrace_frames = []
if exception_stacktrace:
exception_stacktrace_frames = _process_stacktrace_frames(
exception_stacktrace["frames"])
exception_data.pop("stacktrace")
exception = ExceptionModel(**exception_data)
if exception_stacktrace:
del exception_stacktrace["frames"]
stack = Stacktrace(**exception_stacktrace)
stack.frames = exception_stacktrace_frames
exception.stacktrace = stack
exceptions.append(exception)
del data["exception"]
breadcrumbs = []
if data.get("breadcrumbs") and data["breadcrumbs"].get("values"):
for bc_data in data["breadcrumbs"]["values"]:
bc_data["timestamp"] = arrow.get(bc_data["timestamp"])
bc_event_id = bc_data.pop("event_id", None)
if bc_event_id:
# We loop here a few times
# as the event this breadcrumb refers to
# might not have been added yet.
for i in range(3):
bc_event = Event.query.filter_by(
event_id=bc_event_id).first()
if bc_event:
bc_data["related_event_id"] = bc_event.id
break
else:
# This sleep shouldn't matter
# as we are in an `rq-worker`
time.sleep(1+2*i)
bc_level = bc_data.pop("level", None)
if bc_level and hasattr(LogLevels, bc_level.lower()):
bc_data["level"] = getattr(LogLevels, bc_level.lower())
elif bc_level:
bc_data["level"] = bc_level
else:
bc_data["level"] = LogLevels.info
breadcrumbs.append(Breadcrumb(**bc_data))
del data["breadcrumbs"]
stacktrace = None
if data.get("stacktrace"):
if data["stacktrace"]:
frames = _process_stacktrace_frames(
data["stacktrace"]["frames"])
del data["stacktrace"]["frames"]
stacktrace = Stacktrace(**data.pop("stacktrace"))
stacktrace.frames = frames
else:
data.pop("stacktrace")
request_data = None
if data.get("request"):
if data["request"]:
if not data["request"].get("method"):
# This should always be here
# according to the documentation,
# but it doesn't seem to be the case;
# just assume it is a GET request
data["request"]["method"] = "GET"
request_data = RequestModel(**data["request"])
data.pop("request")
template = None
if data.get("template"):
if data["template"]:
template = Template(**data["template"])
data.pop("template")
user = None
if data.get("user"):
if data["user"]:
if data["user"].get("id"):
data["user"]["user_id"] = data["user"]["id"]
del data["user"]["id"]
user = UserModel(**data["user"])
data.pop("user")
if data.get("query"):
if data["query"]:
data["sql_query"] = data["query"]
data.pop("query")
if "sdk" not in data and sentry_client:
sentry_str = sentry_client.split("/")
sentry_sdk = {
"name": sentry_str[0].strip(),
"version": sentry_str[1].strip()
}
data["sdk"] = sentry_sdk
for deprecated in ("time_spent", "site"):
if data.get(deprecated):
data["tags"][deprecated] = data[deprecated]
# TODO: Support these, obviously
for unsupported in ("threads",):
data.pop(unsupported, None)
event = Event(**data)
if exceptions:
event.exceptions = exceptions
if breadcrumbs:
for breadcrumb in breadcrumbs:
breadcrumb.event_id = event.id
event.breadcrumbs = breadcrumbs
if stacktrace:
event.stacktrace = stacktrace
if request_data:
event.request = request_data
if template:
event.template = template
if user:
event.user = user
if not event.message:
if len(event.exceptions) == 1:
event.message = event.exceptions[0].value
else:
# TODO: Allow user to set their own message for events
# This is not ideal,
# as there could be many SDKs deployed to the same platform,
# but returning vastly different information.
if event.stacktrace:
event_type = "Stacktrace"
else:
event_type = "Exception"
culprit = event.culprit or event.server_name or "Unknown"
sdk = "{}/{}".format(data["sdk"].get("name", "Unknown SDK"),
data["sdk"].get("version", "0"))
event.message = "{} from {} ({}) via {}".format(
event_type, culprit, event.platform, sdk)
group = EventGroup.query.filter_by(message=event.message,
project_id=project_id).first()
if not group:
if event.logentry:
event_type = "Log message"
elif event.stacktrace:
event_type = "Stacktrace"
else:
event_type = "Exception"
group = EventGroup(message=event.message, project_id=project_id,
type=event_type)
else:
group.last_seen = datetime.now().isoformat()
db.session.add(group)
event.group = group
db.session.add(event)
db.session.commit()
return event.id
@rq_manager.job()
def send_email_notifications(event_id):
"""
Send email notifications of events.
:param event_id: event ID
:type event_id: int
"""
from cupola.api.run import create_api
from cupola.web import create_app
with create_api().app_context() as ctx:
if not event_id:
ctx.app.logger.warning(gettext("No event ID"))
return
event = Event.query.get(event_id)
if not event:
ctx.app.logger.warning(gettext("No event with ID %(event_id)s",
event_id=event_id))
return
group = event.group
subject = gettext("Event registered for project %(project)s",
project=group.project.name)
# We use `create_app` here so the event URLs are calculated properly
# FIXME: There has to be a nicer way to do this.
with create_app().app_context():
event_url = url_for("events.view_event", event_id=event.event_id,
_external=True)
event_list_url = url_for("events.view_list",
project_id=group.project_id,
_external=True)
if group.project.email_notifications:
messages = []
for member in group.project.email_notifications:
msg = None
if (member.notification_type == "all" or
(member.notification_type == "new" and
group.event_count == 1)):
msg = Message(subject, recipients=[member.user.email])
msg.body = render_template("api/event.txt",
user=member.user,
group=group,
event_url=event_url)
msg.html = render_template("api/event.html",
user=member.user,
group=group,
event_url=event_url)
if member.notification_type == "limit":
event_limit = Event.query.filter(
Event.timestamp >= member.last_notification
).join("group").filter(
EventGroup.project_id == group.project_id,
).count()
if event_limit >= member.notification_limit:
msg = Message(subject,
recipients=[member.user.email])
msg.body = render_template(
"api/event_limit.txt", user=member.user,
group=group, event_count=event_limit,
event_list_url=event_list_url)
msg.html = render_template(
"api/event_limit.html", user=member.user,
group=group, event_count=event_limit,
event_list_url=event_list_url)
if msg:
messages.append(msg)
member.last_notification = datetime.now()
db.session.add(member)
db.session.commit()
if messages:
with mail.connect() as conn:
for message in messages:
conn.send(message)
@rq_manager.job()
def process_data(project_id, payload, sentry_client):
"""
Wrapper to run processing of data.
:param project_id: Project ID to dump data against.
:type project_id: int
:param payload: data sent from the client
:type payload: dict
:param sentry_client: the Sentry client version submitted,
can be None if missing
:type sentry_client: str or None
"""
event_id = record_data(project_id, payload, sentry_client)
send_email_notifications.delay(event_id)

380
cupola/api/view.py

@ -1,7 +1,7 @@
# coding=utf8
#
# views.py: cupola API
# Copyright (C) 2015-2016 Sam Black <samwwwblack@lapwing.org>
# Copyright (C) 2015-2017 Sam Black <samwwwblack@lapwing.org>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
@ -16,36 +16,22 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import base64
import json
import re
import zlib
from datetime import datetime
from urllib.parse import urlparse
from flask import Blueprint
from flask import abort
from flask import make_response
from flask import current_app
from flask import request
from flask import render_template
from flask import url_for
from flask_mail import Message
from cupola.api.queue import rq_manager
from cupola.models import db
from cupola.models.events import DEFAULT_LOG_LEVEL
from cupola.models.events import Event
from cupola.models.events import EventGroup
from cupola.models.events import ExceptionModel
from cupola.models.events import LogLevels
from cupola.models.events import RequestModel
from cupola.models.events import Stacktrace
from cupola.models.events import StacktraceFrames
from cupola.models.events import Template
from cupola.models.events import UserModel
from cupola.models.projects import ClientKeys
from cupola.models.projects import Project
from cupola.utils import mail
from cupola.api.utils import SENTRY_HEADER_AUTH
from cupola.api.utils import ApiAuthError
from cupola.api.utils import ApiProcessError
from cupola.api.utils import get_payload
from cupola.api.utils import payload_event_id
from cupola.api.utils import process_data
api = Blueprint("api", __name__, template_folder="templates")
@ -55,298 +41,88 @@ api = Blueprint("api", __name__, template_folder="templates")
def index(path):
"""
Base API.
:return:
:rtype:
:return: error 404
:rtype: werkzeug.exceptions.Aborter
"""
return abort(404)
def get_payload(data):
"""
"Store" the data
:param data: request data
:type data: str
:return: JSON data loaded into dict
:rtype: dict
"""
try:
raw = zlib.decompress(base64.b64decode(data))
except zlib.error:
try:
raw = zlib.decompress(data)
except zlib.error:
raise abort(403)
if raw:
# Change '.' to u'.'
txt = bytes.decode(raw)
matches = set(re.findall("\"?'?([\w*\.]+\.\w*)'?\"?: ", txt))
for match in matches:
r = match.replace('.', u'\uff0e')
txt = txt.replace(match, r)
return json.loads(txt)
return {}
def _process_stacktrace(stacktraces):
"""
Process and put in order stacktraces.
:param stacktraces: list of stacktrace data
:type stacktraces: list
:return: list of StacktraceFrames
:rtype: list
"""
order = 0
frames = []
for frame in stacktraces:
# Stacktrace frames need to be in order to be meaningful
frame["order"] = order
new_frame = StacktraceFrames(**frame)
frames.append(new_frame)
order += 1
return frames
def record_data(project_id, data):
"""
Record sentry data.
:param project_id: Project ID to dump data against.
:type project_id: int
:param data: data sent from the client
:type data: dict
:return: event ID
:rtype: int
"""
from cupola.api.run import create_api
if project_id != data.pop("project"):
return
with create_api().app_context():
project = Project.query.get(project_id)
if not project:
return
# We do this processing after checking if the project exists
# to possibly save on CPU cycles
for sim in ("sentry.interfaces.Message",
u"sentry\uff0einterfaces\uff0eMessage"):
if sim in data:
# Document field is logentry
data["logentry"] = data[sim]
del data[sim]
break
if data.get("level"):
level = data["level"]
if (level and isinstance(level, str) and
hasattr(LogLevels, level.lower())):
data["level"] = getattr(LogLevels, level.lower())
elif not level:
data["level"] = DEFAULT_LOG_LEVEL
else:
data["level"] = DEFAULT_LOG_LEVEL
if not data.get("logger"):
data["logger"] = "<missing>"
exceptions = []
if data.get("exception"):
for exception_data in data["exception"]["values"]:
exception_stacktrace = None
if exception_data.get("stacktrace"):
if exception_data["stacktrace"]:
exception_stacktrace = _process_stacktrace(
exception_data["stacktrace"]["frames"])
del exception_data["stacktrace"]["frames"]
else:
exception_stacktrace.pop("stacktrace")
exception = ExceptionModel(**exception_data)
if exception_stacktrace:
exception.stacktrace.frames = exception_stacktrace
exceptions.append(exception)
del data["exception"]
stacktrace = None
if data.get("stacktrace"):
if data["stacktrace"]:
frames = _process_stacktrace(data["stacktrace"]["frames"])
del data["stacktrace"]["frames"]
stacktrace = Stacktrace(**data.pop("stacktrace"))
stacktrace.frames = frames
else:
data.pop("stacktrace")
request_data = None
if data.get("request"):
if data["request"]:
request_data = RequestModel(**data["request"])
data.pop("request")
template = None
if data.get("template"):
if data["template"]:
template = Template(**data["template"])
data.pop("template")
user = None
if data.get("user"):
if data["user"]:
if data["user"].get("id"):
data["user"]["user_id"] = data["user"]["id"]
del data["user"]["id"]
user = UserModel(**data["user"])
data.pop("user")
if data.get("query"):
if data["query"]:
data["sql_query"] = data["query"]
data.pop("query")
for deprecated in ("time_spent", "site"):
if data.get(deprecated):
data["tags"][deprecated] = data[deprecated]
event = Event(**data)
if exceptions:
event.exceptions = exceptions
if stacktrace:
event.stacktrace = stacktrace
if request_data:
event.request = request_data
if template:
event.template = template
if user:
event.user = user
group = EventGroup.query.filter_by(message=event.message,
project_id=project_id).first()
if not group:
if event.logentry:
event_type = "Log message"
elif event.stacktrace:
event_type = "Stacktrace"
else:
event_type = "Exception"
group = EventGroup(message=event.message, project_id=project_id,
type=event_type)
else:
group.last_seen = datetime.now().isoformat()
db.session.add(group)
event.group = group
db.session.add(event)
db.session.commit()
return event.id
def send_email_notifications(event_id):
"""
Send email notifications of events.
:param event_id: event ID
:type event_id: int
"""
from cupola.api.run import create_api
from cupola.web import create_app
with create_api().app_context() as ctx:
if not event_id:
ctx.app.logger.warning("No event ID")
return
event = Event.query.get(event_id)
if not event:
ctx.app.logger.warning("No event with ID {}".format(event_id))
return
group = EventGroup.query.filter_by(message=event.message).first()
subject = "Event registered for project {}".format(group.project.name)
# We use `create_app` here so the event URLs are calculated properly
# FIXME: There has to be a nicer way to do this.
with create_app().app_context():
event_url = url_for("events.view_event", event_id=event.event_id,
_external=True)
if group.project.email_notifications:
with mail.connect() as conn:
for user in group.project.email_notifications:
msg = Message(subject, recipients=[user.email])
msg.body = render_template("api/event.txt", user=user,
group=group,
event_url=event_url)
msg.html = render_template("api/event.html", user=user,
group=group,
event_url=event_url)
conn.send(msg)
@rq_manager.job()
def process_data(project_id, payload):
"""
Wrapper to run processing of data.
:param project_id: Project ID to dump data against.
:type project_id: int
:param payload: data sent from the client
:type payload: dict
"""
event_id = record_data(project_id, payload)
send_email_notifications(event_id)
@api.route("/<project_id>/store/", methods=["POST"])
@api.route("/sentry/api/<project_id>/store/", methods=["POST"])
def store(project_id):
"""
Base API.
Base Sentry API.
:param project_id: Project to log error against
:type project_id: str
:return: JSON dump of event ID
:rtype: JSON
"""
sentry_header = request.headers.get("X-Sentry-Auth")
if not sentry_header:
return abort(403)
sentry_public_match = re.search("sentry_key=(\w*)", sentry_header)
sentry_secret_match = re.search("sentry_secret=(\w*)", sentry_header)
if not (sentry_public_match and sentry_secret_match):
return abort(403)
sentry_public = sentry_public_match.group().split("=")[1]
sentry_secret = sentry_secret_match.group().split("=")[1]
# This is only to reject incorrect project or client keys,
# not for use.
ClientKeys.query.filter_by(project_id=project_id, public=sentry_public,
private=sentry_secret).first_or_404()
payload = get_payload(request.data)
sentry_header = request.headers.get(SENTRY_HEADER_AUTH)
origin = request.headers.get("Origin")
referer = request.headers.get("Referer")
if not origin and referer:
parsed_referer = urlparse(referer)
origin = "{}://{}".format(parsed_referer.scheme,
parsed_referer.netloc)
if not (sentry_header or origin or referer):
return abort(ApiAuthError("Unable to find authentication information",
origin))
if not sentry_header and "sentry_key" not in request.args:
return abort(ApiAuthError("Unable to find authentication information",
origin))
if sentry_header:
sentry_public_match = re.search("sentry_key=(\w*)", sentry_header)
sentry_secret_match = re.search("sentry_secret=(\w*)", sentry_header)
if not (sentry_public_match and sentry_secret_match):
return abort(ApiAuthError("Invalid api key", origin))
sentry_public = sentry_public_match.group().split("=")[1]
sentry_secret = sentry_secret_match.group().split("=")[1]
stripped_origin = None
else:
sentry_public = request.args.get("sentry_key")
sentry_secret = None
parsed_url = urlparse(origin)
stripped_origin = parsed_url.netloc if parsed_url.netloc else None
query_stub = ClientKeys.query.filter_by(project_id=project_id,
public=sentry_public)
if sentry_secret:
client_key = query_stub.filter_by(private=sentry_secret).first()
elif origin and stripped_origin:
client_key = query_stub.filter(
ClientKeys.allowed_origins.contains(stripped_origin)).first()
else:
client_key = None
if not client_key:
return abort(ApiAuthError("Invalid api key", origin))
elif client_key and client_key.disabled:
current_app.logger.warning(
"Disabled key use detected: key {} for project {}".format(
client_key.id, client_key.project_id))
# Sentry sends "API key is disabled" here,
# but this allows possible key enumeration.
return abort(ApiAuthError("Invalid api key", origin))
payload = get_payload(request.data, request.content_encoding, origin)
if not payload:
return abort(404)
# TODO: This should probably get what error occurred
return abort(ApiProcessError("Bad data in payload", origin))
# This wraps the job so we can return immediately to the client.
process_data.delay(project_id, payload)
process_data.delay(project_id, payload, request.args.get("sentry_client"))
# Don't add the CORS header stuff if we're responding to a private client
event_resp = payload_event_id(payload,
origin if not sentry_secret else None)
return make_response(json.dumps(payload.get("event_id")))
return event_resp

2
cupola/config.py

@ -53,3 +53,5 @@ REMEMBER_COOKIE_DURATION = timedelta(days=28)
DEBUG_TB_INTERCEPT_REDIRECTS = False
DEBUG_TB_TEMPLATE_EDITOR_ENABLED = True
BABEL_DEFAULT_LOCALE = "en_GB"

42
cupola/forms/projects.py

@ -1,7 +1,7 @@
# coding=utf8
#
# projects.py: Project forms
# Copyright (C) 2015-2016 Sam Black <samwwwblack@lapwing.org>
# Copyright (C) 2015-2017 Sam Black <samwwwblack@lapwing.org>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
@ -16,19 +16,40 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from flask_wtf import Form
from flask_babelex import gettext
from flask_wtf import FlaskForm
from wtforms import Form as WTForm
from wtforms import BooleanField
from wtforms import FormField
from wtforms import IntegerField
from wtforms import SelectField
from wtforms import StringField
from wtforms import SubmitField
from wtforms import TextAreaField
from wtforms.validators import DataRequired
from wtforms_sqlalchemy.fields import QuerySelectField
from cupola.forms.utils import choice_type_coerce_factory
from cupola.models import NOTIFICATION_FREQUENCY
from cupola.models.projects import ProjectUsers
from cupola.models.users import user_query_factory
class ProjectUsersFormBase(FlaskForm):
"""
Per project user settings, for notifications.
"""
notification_type = SelectField(
"Notification frequency", [DataRequired()],
choice_type_coerce_factory(ProjectUsers.notification_type.type),
NOTIFICATION_FREQUENCY)
notification_limit = IntegerField("Notification limit")
class ProjectUsersForm(ProjectUsersFormBase):
submit = SubmitField("Save")
class ProjectSettingsForm(WTForm):
"""
Project settings subform.
@ -36,7 +57,7 @@ class ProjectSettingsForm(WTForm):
scrub_data = BooleanField()
class ProjectForm(Form):
class ProjectForm(FlaskForm):
"""
Project settings form.
"""
@ -48,3 +69,18 @@ class ProjectForm(Form):
settings = FormField(ProjectSettingsForm)
submit = SubmitField("Save")
class ProjectClientKeyForm(FlaskForm):
"""
Project client key
"""
ORIGINS_HELP = gettext(
"List allowed origins, one per line. "
"By default, no origins are allowed to send events.")
name = StringField(validators=[DataRequired()])
allowed_origins = TextAreaField(render_kw={"placeholder": ORIGINS_HELP,
"rows": 5})
submit = SubmitField("Save")

18
cupola/forms/users.py

@ -1,7 +1,7 @@
# coding=utf8
#
# users.py: Cupola user creation/editing
# Copyright (C) 2015-2016 Sam Black <samwwwblack@lapwing.org>
# Copyright (C) 2015-2017 Sam Black <samwwwblack@lapwing.org>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
@ -16,9 +16,10 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from flask_wtf import Form
from flask_wtf import FlaskForm
from wtforms import BooleanField
from wtforms import PasswordField
from wtforms import SelectField
from wtforms import StringField
from wtforms import SubmitField
from wtforms import TextAreaField
@ -26,19 +27,28 @@ from wtforms import validators
from wtforms_sqlalchemy.fields import QuerySelectMultipleField
from cupola.forms.projects import ProjectUsersFormBase
from cupola.forms.utils import LANGUAGES_CHOICE
from cupola.forms.utils import TIMEZONE_CHOICE
from cupola.models.users import role_query_factory
class ProfileForm(Form):
class ProfileForm(FlaskForm):
"""
User profile form.
"""
fullname = StringField("Full name", [validators.DataRequired()])
bio = TextAreaField("Biography")
notification_type = ProjectUsersFormBase.notification_type
notification_limit = ProjectUsersFormBase.notification_limit
locale = SelectField("Locale", choices=LANGUAGES_CHOICE)
timezone = SelectField("Timezone", choices=TIMEZONE_CHOICE)
submit = SubmitField("Save"