관리-도구
편집 파일: spark_driver.cpython-37.opt-1.pyc
B T a! � @ s� d dl mZ d dlmZ d dlmZ d dlmZ d dlm Z e rhd dl m Z d dl m Z d dlm Z mZ G d d � d e�Zd d � Zd d� Zdd� ZG dd� de�ZG dd� de�ZdS )� )�configure_scope)�Hub)� Integration)�capture_internal_exceptions)�MYPY)�Any)�Optional)�Event�Hintc @ s e Zd ZdZedd� �ZdS )�SparkIntegrationZsparkc C s t � d S )N)�patch_spark_context_init� r r �[/opt/alt/python37/lib/python3.7/site-packages/sentry_sdk/integrations/spark/spark_driver.py� setup_once s zSparkIntegration.setup_onceN)�__name__� __module__� __qualname__Z identifier� staticmethodr r r r r r s r c C s6 ddl m} | j}|r2|�d|j� |�d|j� dS )z� Set properties in driver that propagate to worker processes, allowing for workers to have access to those properties. This allows worker integration to have access to app_name and application_id. r )� SparkContextZsentry_app_nameZsentry_application_idN)�pysparkr Z_active_spark_contextZsetLocalProperty�appName� applicationId)r Z spark_contextr r r �_set_app_properties s r c C s4 ddl m} | j}||� t� }| j�� �|� dS )zA Start java gateway server to add custom `SparkListener` r )� ensure_callback_server_startedN)Zpyspark.java_gatewayr Z_gateway�SentryListenerZ_jsc�scZaddSparkListener)r r ZgwZlistenerr r r �_start_sentry_listener( s r c s( ddl m} | j� � fdd�}|| _d S )Nr )r c sZ �� f|�|�}t j�t�d kr$|S t� � t� t� �}|j� fdd��}W d Q R X |S )Nc s t � �� tj�t�d kr | S | �di ��d� �� � | �di ��d� j�d�� | d �d� j�d�� | d �d� j�d �� | d �d � j�d �� | d �d � j � | d �d � j � | d �d� j � | d �d� j � | d �d� j � | �di ��d� j� W d Q R X | S )N�user�idZtagsz executor.idzspark.executor.idzspark-submit.deployModezspark.submit.deployModez driver.hostzspark.driver.hostz driver.portzspark.driver.portZ spark_versionZapp_nameZapplication_id�masterZ spark_homeZextraZweb_url)r r �current�get_integrationr � setdefaultZ sparkUserZ_conf�get�versionr r r Z sparkHomeZuiWebUrl)�eventZhint)�selfr r � process_eventG s( z[patch_spark_context_init.<locals>._sentry_patched_spark_context_init.<locals>.process_event)r r r! r r r r Zadd_event_processor)r& �args�kwargsZinitZscoper' )�spark_context_init)r& r �"_sentry_patched_spark_context_init; s zDpatch_spark_context_init.<locals>._sentry_patched_spark_context_init)r r Z_do_init)r r+ r )r* r r 5 s .r c @ s� e Zd Zdd� Zdd� Zdd� Zdd� Zd d � Zd d � Zd d� Z dd� Z dd� Z dd� Z dd� Z dd� Zdd� Zdd � Zd d � Zd d � Zd!d"� Zd#d$� Zd%d&� Zd'd(� Zd)d*� Zd+d,� Zd-d.� Zd/d0� ZG d1d2� d2�Zd3S )4� SparkListenerc C s d S )Nr )r& ZapplicationEndr r r �onApplicationEndm s z SparkListener.onApplicationEndc C s d S )Nr )r& ZapplicationStartr r r �onApplicationStartq s z SparkListener.onApplicationStartc C s d S )Nr )r& ZblockManagerAddedr r r �onBlockManagerAddedu s z!SparkListener.onBlockManagerAddedc C s d S )Nr )r& ZblockManagerRemovedr r r �onBlockManagerRemovedy s z#SparkListener.onBlockManagerRemovedc C s d S )Nr )r& Z blockUpdatedr r r �onBlockUpdated} s z SparkListener.onBlockUpdatedc C s d S )Nr )r& ZenvironmentUpdater r r �onEnvironmentUpdate� s z!SparkListener.onEnvironmentUpdatec C s d S )Nr )r& Z executorAddedr r r �onExecutorAdded� s z SparkListener.onExecutorAddedc C s d S )Nr )r& ZexecutorBlacklistedr r r �onExecutorBlacklisted� s z#SparkListener.onExecutorBlacklistedc C s d S )Nr )r& ZexecutorBlacklistedForStager r r � onExecutorBlacklistedForStage� s z+SparkListener.onExecutorBlacklistedForStagec C s d S )Nr )r& ZexecutorMetricsUpdater r r �onExecutorMetricsUpdate� s z%SparkListener.onExecutorMetricsUpdatec C s d S )Nr )r& ZexecutorRemovedr r r �onExecutorRemoved� s z SparkListener.onExecutorRemovedc C s d S )Nr )r& �jobEndr r r �onJobEnd� s zSparkListener.onJobEndc C s d S )Nr )r& �jobStartr r r � onJobStart� s zSparkListener.onJobStartc C s d S )Nr )r& ZnodeBlacklistedr r r �onNodeBlacklisted� s z SparkListener.onNodeBlacklistedc C s d S )Nr )r& ZnodeBlacklistedForStager r r �onNodeBlacklistedForStage� s z'SparkListener.onNodeBlacklistedForStagec C s d S )Nr )r& ZnodeUnblacklistedr r r �onNodeUnblacklisted� s z!SparkListener.onNodeUnblacklistedc C s d S )Nr )r& r% r r r � onOtherEvent� s zSparkListener.onOtherEventc C s d S )Nr )r& ZspeculativeTaskr r r �onSpeculativeTaskSubmitted� s z(SparkListener.onSpeculativeTaskSubmittedc C s d S )Nr )r& �stageCompletedr r r �onStageCompleted� s z SparkListener.onStageCompletedc C s d S )Nr )r& �stageSubmittedr r r �onStageSubmitted� s z SparkListener.onStageSubmittedc C s d S )Nr )r& ZtaskEndr r r � onTaskEnd� s zSparkListener.onTaskEndc C s d S )Nr )r& ZtaskGettingResultr r r �onTaskGettingResult� s z!SparkListener.onTaskGettingResultc C s d S )Nr )r& Z taskStartr r r � onTaskStart� s zSparkListener.onTaskStartc C s d S )Nr )r& Z unpersistRDDr r r �onUnpersistRDD� s z SparkListener.onUnpersistRDDc @ s e Zd ZdgZdS )zSparkListener.Javaz1org.apache.spark.scheduler.SparkListenerInterfaceN)r r r Z implementsr r r r �Java� s rI N) r r r r- r. r/ r0 r1 r2 r3 r4 r5 r6 r7 r9 r; r< r= r> r? r@ rB rD rE rF rG rH rI r r r r r, l s2 r, c @ s4 e Zd Zdd� Zdd� Zdd� Zdd� Zd d � Zd S ) r c C s t j| _d S )N)r r �hub)r& r r r �__init__� s zSentryListener.__init__c C s( d� |�� �}| jjd|d� t� d S )NzJob {} Started�info)�level�message)�format�jobIdrJ �add_breadcrumbr )r& r: rN r r r r; � s zSentryListener.onJobStartc C sd d}d}d|� � �� i}|� � �� dkr<d}d�|�� �}nd}d�|�� �}| jj|||d� d S ) N� �resultZ JobSucceededrL z Job {} Ended�warningz Job {} Failed)rM rN �data)Z jobResultZtoStringrO rP rJ rQ )r& r8 rM rN rU r r r r9 � s zSentryListener.onJobEndc C sD |� � }d�|�� �}|�� |�� d�}| jjd||d� t� d S )NzStage {} Submitted)� attemptId�namerL )rM rN rU )� stageInforO �stageIdrV rW rJ rQ r )r& rC � stage_inforN rU r r r rD � s z SentryListener.onStageSubmittedc C s� ddl m} |�� }d}d}|�� |�� d�}y&|�� �� |d< d�|�� �}d}W n& |k rz d�|�� �}d }Y nX | j j |||d � d S ) Nr )� Py4JJavaErrorrR )rV rW �reasonzStage {} FailedrT zStage {} CompletedrL )rM rN rU ) Z py4j.protocolr[ rX rV rW Z failureReasonr# rO rY rJ rQ )r& rA r[ rZ rN rM rU r r r rB � s z SentryListener.onStageCompletedN)r r r rK r; r9 rD rB r r r r r � s r N)Z sentry_sdkr Zsentry_sdk.hubr Zsentry_sdk.integrationsr Zsentry_sdk.utilsr Zsentry_sdk._typesr �typingr r r r r r r r �objectr, r r r r r �<module> s 7g